Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
Refactor AutoGraph and AutoTemporalGraph to use NodeSchema and EdgeSchema for improved type safety and clarity
yifanfeng97191cd744a41a
diff --git a/hyperextract/graphs/base.py b/hyperextract/graphs/base.pyindex eaf6822..7c174fd 100644--- a/hyperextract/graphs/base.py+++ b/hyperextract/graphs/base.py@@ -27,8 +27,8 @@ from ..core.base import BaseAutoTypefrom ..utils.logging import logger-Node = TypeVar("Node", bound=BaseModel)-Edge = TypeVar("Edge", bound=BaseModel)+NodeSchema = TypeVar("NodeSchema", bound=BaseModel)+EdgeSchema = TypeVar("EdgeSchema", bound=BaseModel)# ==============================================================================@@ -66,34 +66,34 @@ DEFAULT_EDGE_PROMPT = ()-class AutoGraphSchema(BaseModel, Generic[Node, Edge]):+class AutoGraphSchema(BaseModel, Generic[NodeSchema, EdgeSchema]):"""Generic schema container for graph-based knowledge patterns."""- nodes: List[Node] = Field(default_factory=list, description="Graph nodes/entities")- edges: List[Edge] = Field(+ nodes: List[NodeSchema] = Field(default_factory=list, description="Graph nodes/entities")+ edges: List[EdgeSchema] = Field(default_factory=list, description="Graph edges/relationships")-class NodeListSchema(BaseModel, Generic[Node]):+class NodeListSchema(BaseModel, Generic[NodeSchema]):"""Intermediate schema for batch node extraction."""- items: List[Node] = Field(+ items: List[NodeSchema] = Field(default_factory=list,description="List of identified entities or nodes found in the text.",)-class EdgeListSchema(BaseModel, Generic[Edge]):+class EdgeListSchema(BaseModel, Generic[EdgeSchema]):"""Intermediate schema for batch edge extraction."""- items: List[Edge] = Field(+ items: List[EdgeSchema] = Field(default_factory=list,description="List of identified relationships or edges found in the text.",)-class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):+class AutoGraph(BaseAutoType[AutoGraphSchema[NodeSchema, EdgeSchema]], Generic[NodeSchema, EdgeSchema]):"""AutoGraph - extracts knowledge graphs with nodes and edges from text.This pattern extracts structured knowledge graphs consisting of entities (nodes) and@@ -134,15 +134,15 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):"""if TYPE_CHECKING:- graph_schema: Type[AutoGraphSchema[Node, Edge]]+ graph_schema: Type[AutoGraphSchema[NodeSchema, EdgeSchema]]def __init__(self,- node_schema: Type[Node],- edge_schema: Type[Edge],- node_key_extractor: Callable[[Node], str],- edge_key_extractor: Callable[[Edge], str],- nodes_in_edge_extractor: Callable[[Edge], Tuple[str, str]],+ node_schema: Type[NodeSchema],+ edge_schema: Type[EdgeSchema],+ node_key_extractor: Callable[[NodeSchema], str],+ edge_key_extractor: Callable[[EdgeSchema], str],+ nodes_in_edge_extractor: Callable[[EdgeSchema], Tuple[str, str]],llm_client: BaseChatModel,embedder: Embeddings,*,@@ -290,7 +290,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):"""Returns the default prompt for one-stage graph extraction."""return DEFAULT_GRAPH_PROMPT- def _create_empty_instance(self) -> "AutoGraph[Node, Edge]":+ def _create_empty_instance(self) -> "AutoGraph[NodeSchema, EdgeSchema]":"""Creates a new empty AutoGraph instance with the same configuration as this one.Overrides parent method to handle AutoGraph-specific parameters.@@ -321,7 +321,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):)@property- def data(self) -> AutoGraphSchema[Node, Edge]:+ def data(self) -> AutoGraphSchema[NodeSchema, EdgeSchema]:"""Returns the current graph state (nodes and edges).Returns:@@ -332,7 +332,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):)@property- def nodes(self) -> List[Node]:+ def nodes(self) -> List[NodeSchema]:"""Returns the current node collection.Returns:@@ -341,7 +341,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):return self._node_memory.items@property- def edges(self) -> List[Edge]:+ def edges(self) -> List[EdgeSchema]:"""Returns the current edge collection.Returns:@@ -369,7 +369,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):self._node_memory.clear_index()self._edge_memory.clear_index()- def _set_data_state(self, data: AutoGraphSchema) -> None:+ def _set_data_state(self, data: AutoGraphSchema[NodeSchema, EdgeSchema]) -> None:"""Replace graph data with new data (full reset).Args:@@ -385,7 +385,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):self.clear_index()- def _update_data_state(self, incoming_data: AutoGraphSchema) -> None:+ def _update_data_state(self, incoming_data: AutoGraphSchema[NodeSchema, EdgeSchema]) -> None:"""Merge incoming graph data into current state.Args:@@ -402,7 +402,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):# ==================== Extraction Pipeline ====================- def _extract_data(self, text: str) -> AutoGraphSchema:+ def _extract_data(self, text: str) -> AutoGraphSchema[NodeSchema, EdgeSchema]:"""Main extraction logic dispatcher.Args:@@ -421,7 +421,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):# Prune dangling edges to ensure graph consistencyreturn self._prune_dangling_edges(raw_graph)- def _extract_data_by_one_stage(self, text: str) -> AutoGraphSchema:+ def _extract_data_by_one_stage(self, text: str) -> AutoGraphSchema[NodeSchema, EdgeSchema]:"""Extract nodes and edges simultaneously using single LLM call.Args:@@ -451,7 +451,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):# Merge multiple graphsreturn self.merge_batch_data(graph_list)- def _extract_data_by_two_stage(self, text: str) -> AutoGraphSchema:+ def _extract_data_by_two_stage(self, text: str) -> AutoGraphSchema[NodeSchema, EdgeSchema]:"""Extract nodes first, then edges with node context (batch processing).Process:@@ -488,7 +488,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):# 5. Global Merge (passes tuples to merge_batch_data)return self.merge_batch_data(partial_graphs)- def _extract_nodes_batch(self, chunks: List[str]) -> List[NodeListSchema[Node]]:+ def _extract_nodes_batch(self, chunks: List[str]) -> List[NodeListSchema[NodeSchema]]:"""Batch extract nodes from multiple text chunks.Args:@@ -508,8 +508,8 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):return llm_chain.batch(inputs, config={"max_concurrency": self.max_workers})def _extract_edges_batch(- self, chunks: List[str], node_lists: List[NodeListSchema[Node]]- ) -> List[EdgeListSchema[Edge]]:+ self, chunks: List[str], node_lists: List[NodeListSchema[NodeSchema]]+ ) -> List[EdgeListSchema[EdgeSchema]]:"""Batch extract edges using corresponding node lists as context.Args:@@ -539,7 +539,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):return llm_chain.batch(inputs, config={"max_concurrency": self.max_workers})- def _prune_dangling_edges(self, graph: AutoGraphSchema) -> AutoGraphSchema:+ def _prune_dangling_edges(self, graph: AutoGraphSchema[NodeSchema, EdgeSchema]) -> AutoGraphSchema[NodeSchema, EdgeSchema]:"""Prune edges that connect to non-existent nodes (Consistency Check).Ensures graph consistency by removing any edges where either endpoint@@ -584,14 +584,14 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):def merge_batch_data(self,- data_list_or_tuple: List[AutoGraphSchema]- | Tuple[List[List[Node]], List[List[Edge]]],- ) -> AutoGraphSchema:+ data_list_or_tuple: List[AutoGraphSchema[NodeSchema, EdgeSchema]]+ | Tuple[List[List[NodeSchema]], List[List[EdgeSchema]]],+ ) -> AutoGraphSchema[NodeSchema, EdgeSchema]:"""Merge multiple graphs or node/edge tuples into one.Supports two input formats:- List of AutoGraphSchema objects (standard format)- - Tuple of (List[List[Node]], List[List[Edge]]) (optimization for batch processing)+ - Tuple of (List[List[NodeSchema]], List[List[EdgeSchema]]) (optimization for batch processing)Args:data_list_or_tuple: Either a list of AutoGraphSchema objects or a tuple of@@ -661,7 +661,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):top_k: int = 3,search_nodes: bool = True,search_edges: bool = True,- ) -> List[Node] | List[Edge] | Tuple[List[Node], List[Edge]]:+ ) -> List[NodeSchema] | List[EdgeSchema] | Tuple[List[NodeSchema], List[EdgeSchema]]:"""Unified graph search interface.By default, searches both nodes and edges simultaneously to provide comprehensive results.@@ -675,9 +675,9 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):search_edges: Whether to search edges (default: True).Returns:- - Tuple[List[Node], List[Edge]] if both search_nodes and search_edges are True (default).- - List[Node] if only search_nodes is True.- - List[Edge] if only search_edges is True.+ - Tuple[List[NodeSchema], List[EdgeSchema]] if both search_nodes and search_edges are True (default).+ - List[NodeSchema] if only search_nodes is True.+ - List[EdgeSchema] if only search_edges is True.Raises:ValueError: If neither search_nodes nor search_edges is True.@@ -708,7 +708,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):edges = self.search_edges(query, top_k)return edges- def search_nodes(self, query: str, top_k: int = 3) -> List[Node]:+ def search_nodes(self, query: str, top_k: int = 3) -> List[NodeSchema]:"""Semantic search for nodes/entities only.Args:@@ -720,7 +720,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):"""return self._node_memory.search(query=query, top_k=top_k)- def search_edges(self, query: str, top_k: int = 3) -> List[Edge]:+ def search_edges(self, query: str, top_k: int = 3) -> List[EdgeSchema]:"""Semantic search for edges/relationships only.Args:diff --git a/hyperextract/graphs/spatial_graph.py b/hyperextract/graphs/spatial_graph.pynew file mode 100644index 0000000..e69de29diff --git a/hyperextract/graphs/temporal_graph.py b/hyperextract/graphs/temporal_graph.pyindex bd5bcb6..eb126be 100644--- a/hyperextract/graphs/temporal_graph.py+++ b/hyperextract/graphs/temporal_graph.py@@ -1,571 +1,292 @@-"""A simple temporal graph implementation using AutoGraph as the core engine."""+"""Generic temporal graph implementation supporting custom schemas with time-aware deduplication."""-from typing import Optional, List+from typing import Type, Callable, Tuple, Any, List, Optionalfrom datetime import datetime-from pydantic import BaseModel, Field-from semhash import SemHashfrom langchain_core.language_models import BaseChatModelfrom langchain_core.prompts import ChatPromptTemplatefrom langchain_core.embeddings import Embeddings-from ontomem.merger import MergeStrategy, CustomRuleMerger+from ontomem.merger import MergeStrategy, BaseMerger-from .base import AutoGraph, AutoGraphSchema-from ..utils.logging import logger+from .base import AutoGraph, NodeSchema, EdgeSchema, NodeListSchema, EdgeListSchema# ==============================================================================-# 1. Schema Definition - Consistent with original AutoTemporalGraph implementation+# Prompt Definition# ==============================================================================+GENERIC_TEMPORAL_NODE_PROMPT = """+You are a professional entity extraction specialist.+Your task is to extract all important entities (Nodes) from the text.-class NodeSchema(BaseModel):- """Schema for representing an entity."""+# Core Principles+1. **Comprehensiveness**: Extract persons, organizations, locations, events, concepts, and other noun-based entities.+2. **Accuracy**: Keep entity names consistent with the source text.+3. **Exclude Time Expressions**: **NEVER** extract dates, years, or time periods (e.g., "2023", "last year", "today") as entity nodes!+ Time is an attribute of relationships, not a node.+4. **Exclude Pure Numbers**: Do not extract standalone amounts or numeric values as independent nodes.- label: str = Field(- description=(- "The semantic category of the entity (e.g., 'Person', 'Event', 'Location', 'Methodology', 'Position') as you understand it from the text. "- "Use 'Relationship' objects if the concept is inherently relational or verbal (e.g., 'plans'). "- "Prefer consistent, single-word categories where possible (e.g., 'Person', not 'Person_Entity'). "- "Do not extract Date entities as they will be integrated in the relation."- )- )- name: str = Field(- description=(- "The unique name or title identifying this entity, representing exactly one concept. "- "For example, 'Yassir', 'CEO', or 'X'. Avoid combining multiple concepts (e.g., 'CEO of X'), "- "since linking them should be done via Relationship objects. "- "Verbs or multi-concept phrases (e.g., 'plans an escape') typically belong in Relationship objects. "- "Do not extract Date entities as they will be integrated in the relation."- )- )---class EdgeSchema(BaseModel):- """Schema for representing a relationship between two entities."""-- startNode: NodeSchema = Field(- description=(- "The 'subject' or source entity of this relationship, which must appear in the EntitiesExtractor."- )- )- endNode: NodeSchema = Field(- description=(- "The 'object' or target entity of this relationship, which must also appear in the EntitiesExtractor."- )- )- name: str = Field(- description=(- "A single, canonical predicate in PRESENT TENSE capturing how the startNode and endNode relate "- "(e.g., 'is_CEO', 'holds_position', 'is_located_in', 'works_at'). "- "ALWAYS use present tense verbs regardless of the temporal context in the text. "- "Avoid compound verbs (e.g., 'plans_and_executes'). "- "If the text implies negation (e.g., 'no longer CEO'), still use the affirmative present form (e.g., 'is_CEO') "- "and rely on 't_end' for the end date. "- "AVOID preposition-only relation names like 'of', 'in', 'at' - use descriptive present-tense verbs instead."- )- )- t_start: Optional[list[str]] = Field(- default_factory=list,- description=(- "A time or interval indicating when this relationship begins or is active. "- "Resolve relative temporal expressions based on the observation_date:\n"- " - 'today' → exact observation_date\n"- " - 'yesterday' → observation_date minus 1 day\n"- " - 'this week' → Monday of observation_date's week\n"- " - 'last week' → Monday of the week before observation_date\n"- " - 'this month' → first day of observation_date's month\n"- " - 'last month' → first day of the month before observation_date\n"- " - 'this year' → January 1st of observation_date's year\n"- " - 'last year' → January 1st of the year before observation_date\n"- "Keep explicit dates as-is (e.g., '18-06-2024'). "- "For example, if 'Yassir became CEO from 2023', then t_start=['01-01-2023']. "- "This can be a single year, a date, or a resolved relative reference. "- "Leave it [] if not specified."- ),- )- t_end: Optional[list[str]] = Field(- default_factory=list,- description=(- "A time or interval indicating when this relationship ceases to hold. "- "Resolve relative temporal expressions based on the observation_date using the same rules as t_start:\n"- " - 'today' → exact observation_date\n"- " - 'yesterday' → observation_date minus 1 day\n"- " - 'this week' → Monday of observation_date's week\n"- " - etc. (same resolution rules as t_start)\n"- "Keep explicit dates as-is. "- "For example, if 'Yassir left his position in 2025', then t_end=['01-01-2025']. "- "Use this field to capture any 'end action' (e.g., leaving a job, ending a marriage), "- "while keeping the relationship name in a canonical present tense form (e.g., 'is_CEO' not 'was_CEO'). "- "Leave it [] if no end date/time is given."- ),- )- t_obs: Optional[list[str]] = Field(- default=None,- description=(- "DO NOT EXTRACT OR INFER THIS FIELD. Leave it null/None. "- "This field represents the extraction timestamp and will be populated manually by the user post-processing."- ),- )- AutoTemporalGraphic_facts: Optional[list[str]] = Field(- default_factory=list,- description=(- "A list of exact string copies of the AutoTemporalGraphic facts or sentences from the source text that provide evidence for this relationship. "- "Select the specific facts from the input context that justify this edge. "- "Do NOT paraphrase or modify the text. Copy it exactly as it appears in the source."- ),- )---# ==============================================================================-# 2. Prompt Definition - Adapted from original AutoTemporalGraph implementation-# ==============================================================================---AutoTemporalGraph_FACTOID_EXTRACTION_PROMPT = """-Observation Date: {observation_date}--You are an expert factoid extraction engine. Your primary function is to read a news paragraph and its associated observation date, and then decompose the text into a comprehensive list of AutoTemporalGraphic, self-contained, and temporally-grounded facts.--## Task-Given an input paragraph and an `observation_date`, generate a list of all distinct factoids present in the text.--## Guidelines for Generating Temporal Factoids--### 1. AutoTemporalGraphic Factoids-- Convert compound or complex sentences into short, single-fact statements-- Each factoid must contain exactly one piece of information or relationship-- Ensure that each factoid is expressed directly and concisely, without redundancies or duplicating the same information across multiple statements-- **Example:** "Unsupervised learning is dedicated to discovering intrinsic patterns in unlabeled datasets" becomes "Unsupervised learning discovers patterns in unlabeled data"--### 2. Decontextualization-- Replace pronouns (e.g., "it," "he," "they") with the full entity name or a clarifying noun phrase-- Include any necessary modifiers so that each factoid is understandable in isolation--### 3. Temporal Context-- Convert ALL time references to absolute dates/times using the observation_date--#### Conversion Rules:-- "today" → exact observation_date-- "yesterday" → observation_date minus 1 day-- "this week" → Monday of observation_date's week-- "last week" → Monday of the week before observation_date-- "this month" → first day of observation_date's month-- "last month" → first day of the month before observation_date-- "this year" → January 1st of observation_date's year-- "last year" → January 1st of the year before observation_date-- Keep explicit dates as-is (e.g., "June 18, 2024")--#### Additional Temporal Guidelines:-- Position time references naturally within factoids-- Split sentences with multiple time references into separate factoids-- **NEVER include relative terms like "today," "yesterday," "last week" in the final factoids**--### 4. Accuracy & Completeness-- Preserve the original meaning without combining multiple facts into a single statement-- Avoid adding details not present in the source text--### 5. End Actions-- If the text indicates the end of a role or an action (for example, someone leaving a position), be explicit about the role/action and the time it ended--### 6. Redundancies-- Eliminate redundancies by simplifying phrases-- **Example:** Convert "the method is crucial for maintaining X" into "the method maintains X"--## Example--**Input:** "On June 18, 2024, Real Madrid won the Champions League final with a 2-1 victory. Following the triumph, fans of Real Madrid celebrated the Champions League victory across the city."--**Output:**-- Real Madrid won the Champions League final on June 18, 2024-- The Champions League final ended with a 2-1 victory for Real Madrid on June 18, 2024-- Fans of Real Madrid celebrated the Champions League victory across the city on June 18, 2024+### Source Text:"""-AutoTemporalGraph_EDGE_EXTRACTION_PROMPT = """-Observation Date: {observation_date}--You are a precise knowledge extraction engine designed to distill unstructured text into a structured Knowledge Graph.-Your goal is to extract all meaningful relationships (edges) between entities, while rigorously capturing their temporal bounds and grounding evidence.--### 🎯 Objective-Extract a list of relationships where each relationship consists of:-1. **Start Node**: The subject entity (Name + Label).-2. **End Node**: The object entity (Name + Label).-3. **Relation Name**: A canonical, present-tense predicate (e.g., 'works_at', 'is_CEO_of').-4. **Time Start (`t_start`)**: Strategies for when the relation began.-5. **Time End (`t_end`)**: Strategies for when the relation ended.-6. **Evidence (`AutoTemporalGraphic_facts`)**: The exact source sentences/facts supporting this edge.--### 📝 Guidelines--#### 1. Entity & Relation Standards-- **Entities**: Extract precise names (e.g., "Apple Inc.", "Steve Jobs"). Assign accurate semantic labels (e.g., "Organization", "Person").-- **Relations**: use **SIMPLE PRESENT TENSE** verbs only (e.g., use 'manages', NOT 'managed', 'was_managing', or 'will_manage').-- **Forbidden**: Do NOT use vague prepositions like "in", "of", "at" as relation names. Use descriptive verbs instead (e.g., "located_in", "part_of").--#### 2. Temporal Extraction (Critical)-- **Format**: All dates must be in `YYYY-MM-DD` or `YYYY` format.-- **Lists**: `t_start` and `t_end` must always be lists of strings. If no date is found, use an empty list `[]`.-- **Relative Time Resolution**: Calculate absolute dates based on the **Observation Date** ({observation_date}).- - "last year" -> {observation_date} year - 1 (Jan 1st)- - "a few months ago" -> Estimate conservatively based on context.- - "currently" -> implies the relationship is active (no `t_end`).-- **End Actions**: If the text says someone "left" or "stopped", capture this in `t_end` while keeping the relation positive (e.g., Relation: "works_at", t_end: ["2023-01-01"]).-- **t_obs**: Always set `t_obs` to null (or empty). Do not try to guess it.--#### 3. Evidence Attribution (AutoTemporalGraphic_facts)-- For every relationship extracted, identify the specific sentence(s) or AutoTemporalGraphic fact(s) in the source text that provide the evidence.-- **COPY EXACTLY**: Populate the `AutoTemporalGraphic_facts` list with the exact strings from the input text.-- **NO Paraphrasing**: Do not summarize or rewrite the source text in this field.-- If a relationship is derived from multiple facts, include all distinct facts in the list.--### 🧠 Few-Shot Examples--**Input**: "Michel served as CFO at Acme Corp from 2019 to 2021. He was hired by Beta Inc in 2021, but left that role in 2023."-**Output**:-- (Michel, is_CFO_of, Acme Corp, t_start=["2019-01-01"], t_end=["2021-01-01"], AutoTemporalGraphic_facts=["Michel served as CFO at Acme Corp from 2019 to 2021."])-- (Michel, works_at, Beta Inc, t_start=["2021-01-01"], t_end=["2023-01-01"], AutoTemporalGraphic_facts=["He was hired by Beta Inc in 2021, but left that role in 2023."])--**Input**: "Sarah was a board member of GreenFuture until 2019."-**Output**:-- (Sarah, is_board_member_of, GreenFuture, t_start=[], t_end=["2019-01-01"], AutoTemporalGraphic_facts=["Sarah was a board member of GreenFuture until 2019."])--**Input**: "(Observation Date: 2024-06-15) John Doe is no longer the CEO of GreenIT since a few months ago."-**Output**:-- (John Doe, is_CEO_of, GreenIT, t_start=[], t_end=["2024-03-15"], AutoTemporalGraphic_facts=["John Doe is no longer the CEO of GreenIT since a few months ago."])- *(Reasoning: "a few months ago" from June is approx. March)*--**Input**: "John Doe's marriage is happening on 26-02-2026."-**Output**:-- (John Doe, has_status, Married, t_start=["2026-02-26"], t_end=[], AutoTemporalGraphic_facts=["John Doe's marriage is happening on 26-02-2026."])--### 🚀 Directives-- Extract strictly what is in the text. Do not hallucinate external facts.-- Use the **Observation Date** as the anchor for all relative temporal expressions.-- Maintain consistency: The same entity name should be used across multiple relations.-- Ensure `AutoTemporalGraphic_facts` contains exact copies of the source text strings.-"""+# Note: This prompt contains {observation_date} placeholder for dynamic injection during extraction+GENERIC_TEMPORAL_EDGE_PROMPT = """+You are an expert temporal knowledge extraction specialist.+Extract meaningful relationships (edges) between the provided entities, paying STRICT attention to temporal correctness.+### Temporal Extraction Rules+Current Observation Date: {observation_date}-# ============================================================================-# Merge Templates for LLM.CUSTOM_RULE Strategy-# ============================================================================+1. **Relative Time Resolution**: You MUST resolve relative time expressions based on the Observation Date.+ - "last year" -> Calculate the year before {observation_date}+ - "yesterday" -> Calculate the date before {observation_date}+ - "currently" -> The relationship is active (implies no end date)+ - "this month" -> First day of the month in {observation_date}+ - "last month" -> First day of the month before {observation_date}+2. **Explicit Dates**: Keep explicit dates (e.g., "2023", "2024-01-01") exactly as written.-EDGE_MERGE_RULE = """You are an intelligent data merging assistant.-You will receive a list of edges representing the exact same relationship (Same Subject -> Same Predicate -> Same Object).+3. **Missing Time**: If no time information is present, leave time fields empty. DO NOT hallucinate dates.-Your task is to merge them into a SINGLE edge object exactly matching the schema.+### General Constraints+1. ONLY extract edges connecting entities from the known entity list provided below.+2. DO NOT create edges involving entities that are not listed.+3. Use the defined schema fields for time (e.g., t_start, t_end, timestamp) as specified in your output format.-Merge strategy:-1. **startNode/endNode**: Keep the node definitions consistent.-2. **name**: Keep the canonical predicate name (since they are already grouped by this key).-3. **t_start**: Collect ALL distinct start dates/times from all inputs into a single list. Remove exact duplicates.-4. **t_end**: Collect ALL distinct end dates/times from all inputs into a single list. Remove exact duplicates.-5. **t_obs**: Collect ALL distinct observation dates from all inputs into a single list.-6. **AutoTemporalGraph_facts**: Merge all evidence strings into a single list. Remove exact duplicate strings."""-# ==============================================================================-# 3. AutoTemporalGraph Implementation - Inherits from AutoGraph-# ==============================================================================-class AutoTemporalGraph(AutoGraph[NodeSchema, EdgeSchema]):"""- AutoTemporalGraph: A specialized AutoGraph for extracting high-quality triple-based KGs.+ Generic Temporal Graph Extractor (AutoTemporalGraph).- Features:- - Two-stage extraction: Extracts AutoTemporalGraphic facts first, then derives edges and nodes from facts- - Customized prompts from original AutoTemporalGraph implementation- - Semantic Deduplication: Includes `match_nodes_and_update_edges` using SemHash with embeddings- - Temporal tracking: t_start, t_end, and t_obs fields capture relationship timing and extraction metadata- - Evidence attribution: AutoTemporalGraphic_facts field traces each extracted edge to source facts- - Nested node schema: Maintains rich semantic information (name + label)+ A flexible implementation supporting user-defined Node and Edge schemas with:+ - **Schema Agnosticism**: Support any user-defined Node and Edge Pydantic models.+ - **Temporal-Aware Deduplication**: Time information is integrated into edge deduplication logic.+ - **Dynamic Time Injection**: Observation Date is injected during extraction for relative time resolution.++ Key Design:+ - `edge_timestamp_extractor`: Function to extract time info from edges (e.g., lambda x: x.year).+ This ensures (A, rel, B) @ 2020 and (A, rel, B) @ 2021 are treated as different edges.+ - Wrapper around `edge_key_extractor`: Combines base relation key with timestamp key.+ - Dynamic Prompt Injection: Observation Date is injected into edge extraction prompts at runtime.Example:- >>> from langchain_openai import ChatOpenAI, OpenAIEmbeddings- >>> llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)- >>> embedder = OpenAIEmbeddings()- >>> kg = AutoTemporalGraph(llm_client=llm, embedder=embedder)-- >>> # 1. Extract: Facts -> Edges -> Nodes- >>> kg.feed_text("Elon Musk is the CEO of SpaceX. Musk leads Tesla.")- >>> print(f"Before dedup - Nodes: {len(kg.nodes)}, Edges: {len(kg.edges)}")-- >>> # 2. Deduplicate: Merges 'Elon Musk' and 'Musk'- >>> kg.match_nodes_and_update_edges(threshold=0.85)- >>> print(f"After dedup - Nodes: {len(kg.nodes)}, Edges: {len(kg.edges)}")+ >>> from pydantic import BaseModel, Field+ >>>+ >>> class MyEntity(BaseModel):+ ... name: str+ ... category: str = "Unknown"+ >>>+ >>> class MyTemporalEdge(BaseModel):+ ... src: str+ ... dst: str+ ... relation: str+ ... year: Optional[str] = None+ >>>+ >>> kg = AutoTemporalGraph(+ ... node_schema=MyEntity,+ ... edge_schema=MyTemporalEdge,+ ... node_key_extractor=lambda x: x.name,+ ... edge_key_extractor=lambda x: f"{x.src}|{x.relation}|{x.dst}",+ ... edge_timestamp_extractor=lambda x: x.year or "",+ ... nodes_in_edge_extractor=lambda x: (x.src, x.dst),+ ... llm_client=llm,+ ... embedder=embedder,+ ... observation_date="2024-01-15"+ ... )"""def __init__(self,+ node_schema: Type[NodeSchema],+ edge_schema: Type[EdgeSchema],+ node_key_extractor: Callable[[NodeSchema], str],+ edge_key_extractor: Callable[[EdgeSchema], str],+ edge_timestamp_extractor: Callable[[EdgeSchema], str],+ nodes_in_edge_extractor: Callable[[EdgeSchema], Tuple[str, str]],llm_client: BaseChatModel,embedder: Embeddings,observation_date: str | None = None,- chunk_size: int = 2000,- chunk_overlap: int = 200,- facts_per_chunk: int = 10,- max_workers: int = 10,- verbose: bool = False,+ extraction_mode: str = "two_stage",+ **kwargs: Any,):- """Initialize AutoTemporalGraph.+ """+ Initialize AutoTemporalGraph.Args:- llm_client: Language model for extraction- embedder: Embedding model for vector indexing- observation_date: Date when the extraction was performed, like '1997-10-10' or '1997-10-10 23:59:59'.- If None, uses current date and time.- chunk_size: Characters per chunk- chunk_overlap: Overlapping characters between chunks- facts_per_chunk: Max number of AutoTemporalGraphic facts to group into a single extraction batch (default: 20)- max_workers: Max concurrent extraction workers- verbose: Display detailed execution logs and progress information+ node_schema: User-defined Node Pydantic model.+ edge_schema: User-defined Edge Pydantic model with time fields.+ node_key_extractor: Function to extract unique key from node (e.g., lambda x: x.name).+ edge_key_extractor: Function to extract base relationship key (e.g., lambda x: f"{x.src}|{x.relation}|{x.dst}").+ edge_timestamp_extractor: Function to extract time fingerprint from edge (e.g., lambda x: x.year).+ This is critical for differentiating edges at different times.+ nodes_in_edge_extractor: Function to extract (source_key, target_key) from edge.+ llm_client: LangChain BaseChatModel for extraction.+ embedder: LangChain Embeddings for semantic operations.+ observation_date: Date context for relative time resolution (default: today in YYYY-MM-DD format).+ extraction_mode: "one_stage" or "two_stage" (default: "two_stage").+ **kwargs: Additional arguments passed to parent AutoGraph (e.g., prompt_for_node_extraction,+ node_strategy_or_merger, edge_strategy_or_merger, etc.)."""-- self.facts_per_chunk = facts_per_chunk- self.observation_date = observation_date-- # 1. Define Key Extractors (critical for deduplication)- # Node deduplication: exact match by name- node_key_fn = lambda x: x.name-- # Edge deduplication: combination of subject-predicate-object triple- edge_key_fn = lambda x: f"{x.startNode.name}|{x.name}|{x.endNode.name}"-- # 2. Edge consistency check: tell AutoGraph which nodes this edge connects- nodes_in_edge_fn = lambda x: (x.startNode.name, x.endNode.name)-- # Edge merger: merges relationship descriptions using EDGE_MERGE_RULE- edge_merger = CustomRuleMerger(- key_extractor=edge_key_fn,- llm_client=llm_client,- item_schema=EdgeSchema,- rule=EDGE_MERGE_RULE,- )-- # 3. Call parent class initialization- logger.info("🚀 Initializing AutoTemporalGraph")+ # Set observation date (default: today)+ self.observation_date = observation_date or datetime.now().strftime("%Y-%m-%d")++ # Store raw extractors for instance recreation (_create_empty_instance)+ self.raw_timestamp_extractor = edge_timestamp_extractor+ self.raw_edge_key_extractor = edge_key_extractor++ # -----------------------------------------------------------+ # Core Logic 1: Decorate Edge Key Extractor+ # -----------------------------------------------------------+ # Combine "base relation fingerprint" (e.g., A|rel|B) with "time fingerprint" (e.g., 2023)+ # This ensures OMem treats (A, rel, B) @ t1 and (A, rel, B) @ t2 as different records+ def temporal_edge_key_wrapper(edge: EdgeSchema) -> str:+ base_key = edge_key_extractor(edge)+ time_key = edge_timestamp_extractor(edge)+ # Only append timestamp if valid time info exists+ if time_key:+ return f"{base_key}|{time_key}"+ return base_key++ # -----------------------------------------------------------+ # Core Logic 2: Prepare Edge Extraction Prompt+ # -----------------------------------------------------------+ # Extract user-provided edge prompt or use default+ edge_prompt = kwargs.pop("prompt_for_edge_extraction", GENERIC_TEMPORAL_EDGE_PROMPT)++ # Store the template (with {observation_date} placeholder) for later injection+ self.temporal_edge_prompt_template = edge_prompt++ # Initialize parent AutoGraphsuper().__init__(- node_schema=NodeSchema,- edge_schema=EdgeSchema,- node_key_extractor=node_key_fn,- edge_key_extractor=edge_key_fn,- nodes_in_edge_extractor=nodes_in_edge_fn,+ node_schema=node_schema,+ edge_schema=edge_schema,+ node_key_extractor=node_key_extractor,+ edge_key_extractor=temporal_edge_key_wrapper, # Inject wrapped extractor+ nodes_in_edge_extractor=nodes_in_edge_extractor,llm_client=llm_client,embedder=embedder,- # Enforce one-stage extraction (first extract edges, then derive nodes)- extraction_mode="one_stage",- # Inject customized prompts- prompt_for_edge_extraction=AutoTemporalGraph_EDGE_EXTRACTION_PROMPT,- # Configure deduplication strategy- node_strategy_or_merger=MergeStrategy.KEEP_EXISTING,- edge_strategy_or_merger=edge_merger,- # Optimize indexing: only index name field- node_fields_for_index=["name", "label"],- edge_fields_for_index=["startNode", "name", "endNode"],- # Other parameters- chunk_size=chunk_size,- chunk_overlap=chunk_overlap,- max_workers=max_workers,- verbose=verbose,+ extraction_mode=extraction_mode,+ prompt_for_edge_extraction=edge_prompt, # Pass template with placeholder+ # Use default node prompt if not provided+ prompt_for_node_extraction=kwargs.pop(+ "prompt_for_node_extraction", GENERIC_TEMPORAL_NODE_PROMPT+ ),+ **kwargs,)- # ==================== Extraction Pipeline ====================+ # ==============================================================================+ # Override Extraction Methods to Dynamically Inject Observation Date+ # ==============================================================================- def _extract_data(self, text: str) -> AutoGraphSchema:- """Extract AutoTemporalGraphic facts first, then extract edges (Two-stage extraction).+ def _extract_edges_batch(+ self, chunks: List[str], node_lists: List[NodeListSchema[NodeSchema]]+ ) -> List[EdgeListSchema[EdgeSchema]]:+ """+ Override: Inject observation_date into edge extraction chain during two-stage extraction.- Process:- 1. Split text into chunks.- 2. Batch extract AutoTemporalGraphic facts from chunks.- 3. Consolidate facts into a unified context.- 4. Split consolidated facts into chunks.- 5. Batch extract edges from fact chunks.- 6. Post-process: Set t_obs timestamp & Derive nodes.- 7. Merge all partial graphs into one global graph.+ This method is called during the second stage (edge extraction from nodes + text).+ It dynamically injects the observation_date into the LLM chain inputs.Args:- text: Input text.+ chunks: List of text chunks.+ node_lists: List of extracted node schemas for each chunk.Returns:- Extracted and validated graph.+ List of extracted edge schemas."""- obs_date_str = self.observation_date or datetime.now().strftime("%Y-%m-%d")-- # ==================== Step 1: Extract AutoTemporalGraphic Facts ====================- logger.info("🔍 [Phase 1] Extracting AutoTemporalGraphic Facts...")-- # 1. Prepare raw chunks- if len(text) <= self.chunk_size:- raw_chunks = [text]- else:- raw_chunks = self.text_splitter.split_text(text)-- # ==================== Step 2: Extract Edges from Facts ====================- logger.info("🔍 [Phase 2] Extracting Edges from AutoTemporalGraphic Facts...")-- # 5. Group Facts into Chunks (Controlled by count AND size)- # This ensures each fact remains intact and fact IDs reset per chunk- fact_chunks = []- current_batch = []- current_char_count = 0-- # Estimate overhead: "[Fact 99]: " + newline is approx 12 chars- PREFIX_OVERHEAD = 12-- for fact in raw_chunks:- fact_len = len(fact) + PREFIX_OVERHEAD-- # Check constraints:- # 1. Size limit (fuzzy check to keep under chunk_size)- # 2. Count limit (facts_per_chunk)- size_limit_reached = (current_char_count + fact_len) > self.chunk_size- count_limit_reached = len(current_batch) >= self.facts_per_chunk-- if (size_limit_reached or count_limit_reached) and current_batch:- # Flush current batch- formatted_chunk = "\n".join(- [f"[Fact {i + 1}]: {f}" for i, f in enumerate(current_batch)]- )- fact_chunks.append(formatted_chunk)-- # Reset- current_batch = []- current_char_count = 0-- current_batch.append(fact)- current_char_count += fact_len-- # Flush remaining items- if current_batch:- formatted_chunk = "\n".join(- [f"[Fact {i + 1}]: {f}" for i, f in enumerate(current_batch)]+ inputs = []+ for chunk, node_list in zip(chunks, node_lists):+ nodes = node_list.items if node_list else []+ if not nodes:+ node_context = "No specific entities identified in this chunk."+ else:+ node_keys = [self.node_key_extractor(n) for n in nodes]+ node_context = "Known entities: " + ", ".join(node_keys)++ # Build input dictionary with dynamic observation_date injection+ inputs.append(+ {+ "chunk_text": chunk,+ "node_context": node_context,+ "observation_date": self.observation_date, # <--- DYNAMIC INJECTION+ })- fact_chunks.append(formatted_chunk)- # 6. Batch Extract Edges directly from Fact Chunks- # Format the edge prompt with observation date- edge_prompt_with_date = self.edge_prompt.format(observation_date=obs_date_str)- edge_prompt_template = ChatPromptTemplate.from_template(- f"{edge_prompt_with_date}\n\n### Source Text:\n{{chunk_text}}"+ # Create prompt template with placeholder for observation_date+ # The self.temporal_edge_prompt_template contains {observation_date} which will be filled+ prompt_template = ChatPromptTemplate.from_template(+ f"{self.temporal_edge_prompt_template}{{node_context}}\n\n### Source Text:\n{{chunk_text}}")- edge_chain = edge_prompt_template | self.llm_client.with_structured_output(+ llm_chain = prompt_template | self.llm_client.with_structured_output(self.edge_list_schema)- edge_inputs = [{"chunk_text": chunk} for chunk in fact_chunks]- chunk_edge_lists = edge_chain.batch(- edge_inputs, config={"max_concurrency": self.max_workers}- )-- # ==================== Step 3: Post-process & Merge ====================-- # 8. Post-process: Derive Nodes & Set t_obs- all_nodes_lists, all_edges_lists = [], []-- for edge_list in chunk_edge_lists:- derived_nodes, processed_edges = [], []+ return llm_chain.batch(inputs, config={"max_concurrency": self.max_workers})- if edge_list and edge_list.items:- for edge in edge_list.items:- edge: EdgeSchema- # Set t_obs to extraction timestamp- edge.t_obs = [obs_date_str]-- # Derive nodes from edges- derived_nodes.append(edge.startNode)- derived_nodes.append(edge.endNode)- processed_edges.append(edge)-- all_nodes_lists.append(derived_nodes)- all_edges_lists.append(processed_edges)-- # 9. Construct Partial Graphs- partial_graphs = (all_nodes_lists, all_edges_lists)-- # 10. Global Merge- raw_graph = self.merge_batch_data(partial_graphs)-- return self._prune_dangling_edges(raw_graph)-- # ==================== Node Deduplication ====================-- def match_nodes_and_update_edges(- self, threshold: float = 0.8- ) -> "AutoTemporalGraph":- """Match nodes in the graph and update edges accordingly using SemHash with embeddings.-- This method identifies and merges similar nodes based on semantic similarity- using embeddings from the instance's embedder. It updates edges to reflect- any changes in node identities.+ def _extract_data_by_one_stage(self, text: str) -> Any:+ """+ Override: Inject observation_date into one-stage extraction if the prompt contains the placeholder.Args:- threshold: Similarity threshold for matching nodes (0.0 to 1.0). Defaults to 0.8.+ text: Input text to extract.Returns:- The updated AutoTemporalGraph instance with matched nodes and updated edges.+ Extracted graph."""- logger.info(- f"🔄 Starting node matching and edge update (threshold={threshold})..."- )+ # Check if prompt template contains observation_date placeholder+ needs_date = "{observation_date}" in self.prompt- nodes, edges = self.nodes, self.edges+ template_str = f"{self.prompt}{{chunk_text}}"+ prompt_template = ChatPromptTemplate.from_template(template_str)+ llm_chain = prompt_template | self.llm_client.with_structured_output(+ self.graph_schema+ )- if not nodes:- logger.warning("⚠️ No nodes to match; skipping matching.")- return self+ # Prepare base input+ base_input = {}+ if needs_date:+ base_input["observation_date"] = self.observation_date- # 1. Generate Embeddings using self.embedder.embed_documents- node_names = [n.name for n in nodes]- logger.info(f"📊 Generating embeddings for {len(node_names)} nodes...")+ # Process text (chunked if necessary)+ if len(text) <= self.chunk_size:+ inp = {"chunk_text": text, **base_input}+ graph = llm_chain.invoke(inp)+ graph_list = [graph]+ else:+ chunks = self.text_splitter.split_text(text)+ inputs = [{"chunk_text": chunk, **base_input} for chunk in chunks]+ graph_list = llm_chain.batch(+ inputs, config={"max_concurrency": self.max_workers}+ )- try:- embeddings = self.embedder.embed_documents(node_names)- except Exception as e:- logger.error(f"❌ Failed to generate embeddings: {e}")- return self+ return self.merge_batch_data(graph_list)- # 2. Initialize SemHash from embeddings- semhash = SemHash.from_embeddings(- embeddings=embeddings,- records=node_names,- model=None, # Embeddings already provided- )+ def _create_empty_instance(self) -> "AutoTemporalGraph[NodeSchema, EdgeSchema]":+ """+ Override: Recreate instance with all temporal-specific attributes.- # 3. Run Deduplication- result = semhash.self_deduplicate(threshold=threshold)-- # 4. Build Mapping from duplicates result- mapping = {} # old_name -> new_name- for record in result.filtered:- if record.duplicates and len(record.duplicates) > 0:- # Map the record to the first duplicate (representative)- mapping[record.record] = record.duplicates[0][0]-- if not mapping:- logger.info("✅ No similar nodes found above threshold.")- return self-- logger.info(f"🔗 Found {len(mapping)} nodes to merge.")-- # 5. Apply mapping to graph data- # Update Node names- for node in nodes:- if node.name in mapping:- node.name = mapping[node.name]-- # Update Edge references (startNode and endNode are nested NodeSchema objects)- for edge in edges:- if edge.startNode.name in mapping:- edge.startNode.name = mapping[edge.startNode.name]- if edge.endNode.name in mapping:- edge.endNode.name = mapping[edge.endNode.name]-- # 6. Update internal state- new_data = AutoGraphSchema(nodes=nodes, edges=edges)- self._set_data_state(new_data)-- logger.info(- f"✅ Node matching complete: Nodes {len(nodes)} -> {len(self.nodes)}, Edges: {len(edges)}"+ This is called by the parent class to create new instances during certain operations.+ We need to preserve raw extractors and temporal configuration.+ """+ return self.__class__(+ node_schema=self.node_schema,+ edge_schema=self.edge_schema,+ node_key_extractor=self.node_key_extractor,+ edge_key_extractor=self.raw_edge_key_extractor, # Pass original (unwrapped)+ edge_timestamp_extractor=self.raw_timestamp_extractor, # Pass original+ nodes_in_edge_extractor=self.nodes_in_edge_extractor,+ llm_client=self.llm_client,+ embedder=self.embedder,+ observation_date=self.observation_date,+ extraction_mode=self.extraction_mode,+ node_strategy_or_merger=self.node_merger,+ edge_strategy_or_merger=self.edge_merger,+ prompt_for_node_extraction=self.node_prompt,+ prompt_for_edge_extraction=self.temporal_edge_prompt_template,+ chunk_size=self.chunk_size,+ chunk_overlap=self.chunk_overlap,+ max_workers=self.max_workers,+ verbose=self.verbose,+ node_fields_for_index=self.node_fields_for_index,+ edge_fields_for_index=self.edge_fields_for_index,)- return self