Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
``` feat(graphs): 添加Atom知识图谱提取实现
yifanfeng97dac1395e10f7
diff --git a/hyperextract/graphs/atom.py b/hyperextract/graphs/atom.pyindex e69de29..5559d03 100644--- a/hyperextract/graphs/atom.py+++ b/hyperextract/graphs/atom.py@@ -0,0 +1,553 @@+"""Atom implementation using AutoGraph as the core engine.++This module provides a specialized implementation of the Atom algorithm+using the AutoGraph framework, designed for extracting high-quality,+standardized triple-based knowledge graphs.++Prompts and schemas are adapted from the original Atom implementation.+"""++from typing import Optional+from datetime import datetime+from pydantic import BaseModel, Field+from semhash import SemHash+from langchain_core.language_models import BaseChatModel+from langchain_core.prompts import ChatPromptTemplate+from langchain_core.embeddings import Embeddings+from ontomem.merger import MergeStrategy++from .base import AutoGraph, AutoGraphSchema+from ..utils.logging import logger++# ==============================================================================+# 1. Schema Definition - Consistent with original Atom implementation+# ==============================================================================+++class AtomicFactSchema(BaseModel):+ """Schema for extracting atomic facts from text."""++ atomic_fact: list[str] = Field(+ description=(+ "A comprehensive list of atomic, self-contained, and temporally-grounded facts extracted from the text."+ )+ )+++class NodeSchema(BaseModel):+ """Schema for representing an entity."""++ label: str = Field(+ description=(+ "The semantic category of the entity (e.g., 'Person', 'Event', 'Location', 'Methodology', 'Position') as you understand it from the text. "+ "Use 'Relationship' objects if the concept is inherently relational or verbal (e.g., 'plans'). "+ "Prefer consistent, single-word categories where possible (e.g., 'Person', not 'Person_Entity'). "+ "Do not extract Date entities as they will be integrated in the relation."+ )+ )+ name: str = Field(+ description=(+ "The unique name or title identifying this entity, representing exactly one concept. "+ "For example, 'Yassir', 'CEO', or 'X'. Avoid combining multiple concepts (e.g., 'CEO of X'), "+ "since linking them should be done via Relationship objects. "+ "Verbs or multi-concept phrases (e.g., 'plans an escape') typically belong in Relationship objects. "+ "Do not extract Date entities as they will be integrated in the relation."+ )+ )+++class EdgeSchema(BaseModel):+ """Schema for representing a relationship between two entities."""++ startNode: NodeSchema = Field(+ description=(+ "The 'subject' or source entity of this relationship, which must appear in the EntitiesExtractor."+ )+ )+ endNode: NodeSchema = Field(+ description=(+ "The 'object' or target entity of this relationship, which must also appear in the EntitiesExtractor."+ )+ )+ name: str = Field(+ description=(+ "A single, canonical predicate in PRESENT TENSE capturing how the startNode and endNode relate "+ "(e.g., 'is_CEO', 'holds_position', 'is_located_in', 'works_at'). "+ "ALWAYS use present tense verbs regardless of the temporal context in the text. "+ "Avoid compound verbs (e.g., 'plans_and_executes'). "+ "If the text implies negation (e.g., 'no longer CEO'), still use the affirmative present form (e.g., 'is_CEO') "+ "and rely on 't_end' for the end date. "+ "AVOID preposition-only relation names like 'of', 'in', 'at' - use descriptive present-tense verbs instead."+ )+ )+ t_start: Optional[list[str]] = Field(+ default_factory=list,+ description=(+ "A time or interval indicating when this relationship begins or is active. "+ "Resolve relative temporal expressions based on the observation_date:\n"+ " - 'today' → exact observation_date\n"+ " - 'yesterday' → observation_date minus 1 day\n"+ " - 'this week' → Monday of observation_date's week\n"+ " - 'last week' → Monday of the week before observation_date\n"+ " - 'this month' → first day of observation_date's month\n"+ " - 'last month' → first day of the month before observation_date\n"+ " - 'this year' → January 1st of observation_date's year\n"+ " - 'last year' → January 1st of the year before observation_date\n"+ "Keep explicit dates as-is (e.g., '18-06-2024'). "+ "For example, if 'Yassir became CEO from 2023', then t_start=['01-01-2023']. "+ "This can be a single year, a date, or a resolved relative reference. "+ "Leave it [] if not specified."+ ),+ )+ t_end: Optional[list[str]] = Field(+ default_factory=list,+ description=(+ "A time or interval indicating when this relationship ceases to hold. "+ "Resolve relative temporal expressions based on the observation_date using the same rules as t_start:\n"+ " - 'today' → exact observation_date\n"+ " - 'yesterday' → observation_date minus 1 day\n"+ " - 'this week' → Monday of observation_date's week\n"+ " - etc. (same resolution rules as t_start)\n"+ "Keep explicit dates as-is. "+ "For example, if 'Yassir left his position in 2025', then t_end=['01-01-2025']. "+ "Use this field to capture any 'end action' (e.g., leaving a job, ending a marriage), "+ "while keeping the relationship name in a canonical present tense form (e.g., 'is_CEO' not 'was_CEO'). "+ "Leave it [] if no end date/time is given."+ ),+ )+ t_obs: Optional[list[str]] = Field(+ default=None,+ description=(+ "DO NOT EXTRACT OR INFER THIS FIELD. Leave it null/None. "+ "This field represents the extraction timestamp and will be populated manually by the user post-processing."+ ),+ )+ atomic_facts: Optional[list[str]] = Field(+ default_factory=list,+ description=(+ "A list of exact string copies of the atomic facts or sentences from the source text that provide evidence for this relationship. "+ "Select the specific facts from the input context that justify this edge. "+ "Do NOT paraphrase or modify the text. Copy it exactly as it appears in the source."+ )+ )+++# ==============================================================================+# 2. Prompt Definition - Adapted from original Atom implementation+# ==============================================================================+++Atom_FACTOID_EXTRACTION_PROMPT = """+You are an expert factoid extraction engine. Your primary function is to read a news paragraph and its associated observation date, and then decompose the text into a comprehensive list of atomic, self-contained, and temporally-grounded facts.++## Task+Given an input paragraph and an `observation_date`, generate a list of all distinct factoids present in the text.++## Guidelines for Generating Temporal Factoids++### 1. Atomic Factoids+- Convert compound or complex sentences into short, single-fact statements+- Each factoid must contain exactly one piece of information or relationship+- Ensure that each factoid is expressed directly and concisely, without redundancies or duplicating the same information across multiple statements+- **Example:** "Unsupervised learning is dedicated to discovering intrinsic patterns in unlabeled datasets" becomes "Unsupervised learning discovers patterns in unlabeled data"++### 2. Decontextualization+- Replace pronouns (e.g., "it," "he," "they") with the full entity name or a clarifying noun phrase+- Include any necessary modifiers so that each factoid is understandable in isolation++### 3. Temporal Context+- Convert ALL time references to absolute dates/times using the observation_date++#### Conversion Rules:+- "today" → exact observation_date+- "yesterday" → observation_date minus 1 day+- "this week" → Monday of observation_date's week+- "last week" → Monday of the week before observation_date+- "this month" → first day of observation_date's month+- "last month" → first day of the month before observation_date+- "this year" → January 1st of observation_date's year+- "last year" → January 1st of the year before observation_date+- Keep explicit dates as-is (e.g., "June 18, 2024")++#### Additional Temporal Guidelines:+- Position time references naturally within factoids+- Split sentences with multiple time references into separate factoids+- **NEVER include relative terms like "today," "yesterday," "last week" in the final factoids**++### 4. Accuracy & Completeness+- Preserve the original meaning without combining multiple facts into a single statement+- Avoid adding details not present in the source text++### 5. End Actions+- If the text indicates the end of a role or an action (for example, someone leaving a position), be explicit about the role/action and the time it ended++### 6. Redundancies+- Eliminate redundancies by simplifying phrases+- **Example:** Convert "the method is crucial for maintaining X" into "the method maintains X"++## Example++**Input:** "On June 18, 2024, Real Madrid won the Champions League final with a 2-1 victory. Following the triumph, fans of Real Madrid celebrated the Champions League victory across the city."++**Output:**+- Real Madrid won the Champions League final on June 18, 2024+- The Champions League final ended with a 2-1 victory for Real Madrid on June 18, 2024+- Fans of Real Madrid celebrated the Champions League victory across the city on June 18, 2024+"""++Atom_EDGE_EXTRACTION_PROMPT = """+Observation Date: {obs_timestamp}++You are a precise knowledge extraction engine designed to distill unstructured text into a structured Knowledge Graph.+Your goal is to extract all meaningful relationships (edges) between entities, while rigorously capturing their temporal bounds and grounding evidence.++### 🎯 Objective+Extract a list of relationships where each relationship consists of:+1. **Start Node**: The subject entity (Name + Label).+2. **End Node**: The object entity (Name + Label).+3. **Relation Name**: A canonical, present-tense predicate (e.g., 'works_at', 'is_CEO_of').+4. **Time Start (`t_start`)**: Strategies for when the relation began.+5. **Time End (`t_end`)**: Strategies for when the relation ended.+6. **Evidence (`atomic_facts`)**: The exact source sentences/facts supporting this edge.++### 📝 Guidelines++#### 1. Entity & Relation Standards+- **Entities**: Extract precise names (e.g., "Apple Inc.", "Steve Jobs"). Assign accurate semantic labels (e.g., "Organization", "Person").+- **Relations**: use **SIMPLE PRESENT TENSE** verbs only (e.g., use 'manages', NOT 'managed', 'was_managing', or 'will_manage').+- **Forbidden**: Do NOT use vague prepositions like "in", "of", "at" as relation names. Use descriptive verbs instead (e.g., "located_in", "part_of").++#### 2. Temporal Extraction (Critical)+- **Format**: All dates must be in `YYYY-MM-DD` or `YYYY` format.+- **Lists**: `t_start` and `t_end` must always be lists of strings. If no date is found, use an empty list `[]`.+- **Relative Time Resolution**: Calculate absolute dates based on the **Observation Date** ({obs_timestamp}).+ - "last year" -> {obs_timestamp} year - 1 (Jan 1st)+ - "a few months ago" -> Estimate conservatively based on context.+ - "currently" -> implies the relationship is active (no `t_end`).+- **End Actions**: If the text says someone "left" or "stopped", capture this in `t_end` while keeping the relation positive (e.g., Relation: "works_at", t_end: ["2023-01-01"]).+- **t_obs**: Always set `t_obs` to null (or empty). Do not try to guess it.++#### 3. Evidence Attribution (atomic_facts)+- For every relationship extracted, identify the specific sentence(s) or atomic fact(s) in the source text that provide the evidence.+- **COPY EXACTLY**: Populate the `atomic_facts` list with the exact strings from the input text.+- **NO Paraphrasing**: Do not summarize or rewrite the source text in this field.+- If a relationship is derived from multiple facts, include all distinct facts in the list.++### 🧠 Few-Shot Examples++**Input**: "Michel served as CFO at Acme Corp from 2019 to 2021. He was hired by Beta Inc in 2021, but left that role in 2023."+**Output**:+- (Michel, is_CFO_of, Acme Corp, t_start=["2019-01-01"], t_end=["2021-01-01"], atomic_facts=["Michel served as CFO at Acme Corp from 2019 to 2021."])+- (Michel, works_at, Beta Inc, t_start=["2021-01-01"], t_end=["2023-01-01"], atomic_facts=["He was hired by Beta Inc in 2021, but left that role in 2023."])++**Input**: "Sarah was a board member of GreenFuture until 2019."+**Output**:+- (Sarah, is_board_member_of, GreenFuture, t_start=[], t_end=["2019-01-01"], atomic_facts=["Sarah was a board member of GreenFuture until 2019."])++**Input**: "(Observation Date: 2024-06-15) John Doe is no longer the CEO of GreenIT since a few months ago."+**Output**:+- (John Doe, is_CEO_of, GreenIT, t_start=[], t_end=["2024-03-15"], atomic_facts=["John Doe is no longer the CEO of GreenIT since a few months ago."])+ *(Reasoning: "a few months ago" from June is approx. March)*++**Input**: "John Doe's marriage is happening on 26-02-2026."+**Output**:+- (John Doe, has_status, Married, t_start=["2026-02-26"], t_end=[], atomic_facts=["John Doe's marriage is happening on 26-02-2026."])++### 🚀 Directives+- Extract strictly what is in the text. Do not hallucinate external facts.+- Use the **Observation Date** as the anchor for all relative temporal expressions.+- Maintain consistency: The same entity name should be used across multiple relations.+- Ensure `atomic_facts` contains exact copies of the source text strings.+"""+++# ==============================================================================+# 3. Atom Implementation - Inherits from AutoGraph+# ==============================================================================+++class Atom(AutoGraph[NodeSchema, EdgeSchema]):+ """+ Atom: A specialized AutoGraph for extracting high-quality triple-based KGs.++ Features:+ - Two-stage extraction: Extracts atomic facts first, then derives edges and nodes from facts+ - Customized prompts from original Atom implementation+ - Semantic Deduplication: Includes `match_nodes_and_update_edges` using SemHash with embeddings+ - Temporal tracking: t_start, t_end, and t_obs fields capture relationship timing and extraction metadata+ - Evidence attribution: atomic_facts field traces each extracted edge to source facts+ - Nested node schema: Maintains rich semantic information (name + label)++ Example:+ >>> from langchain_openai import ChatOpenAI, OpenAIEmbeddings+ >>> llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)+ >>> embedder = OpenAIEmbeddings()+ >>> kg = Atom(llm_client=llm, embedder=embedder)++ >>> # 1. Extract: Facts -> Edges -> Nodes+ >>> kg.feed_text("Elon Musk is the CEO of SpaceX. Musk leads Tesla.")+ >>> print(f"Before dedup - Nodes: {len(kg.nodes)}, Edges: {len(kg.edges)}")++ >>> # 2. Deduplicate: Merges 'Elon Musk' and 'Musk'+ >>> kg.match_nodes_and_update_edges(threshold=0.85)+ >>> print(f"After dedup - Nodes: {len(kg.nodes)}, Edges: {len(kg.edges)}")+ """++ def __init__(+ self,+ llm_client: BaseChatModel,+ embedder: Embeddings,+ observation_date: str | None = None,+ chunk_size: int = 2000,+ chunk_overlap: int = 200,+ max_workers: int = 10,+ verbose: bool = False,+ ):+ """Initialize Atom.++ Args:+ llm_client: Language model for extraction+ embedder: Embedding model for vector indexing+ observation_date: Date when the extraction was performed, like '1997-10-10' or '1997-10-10 23:59:59'.+ If None, uses current date and time.+ chunk_size: Characters per chunk+ chunk_overlap: Overlapping characters between chunks+ max_workers: Max concurrent extraction workers+ verbose: Display detailed execution logs and progress information+ """++ self.observation_date = observation_date++ # 1. Define Key Extractors (critical for deduplication)+ # Node deduplication: exact match by name+ node_key_fn = lambda x: x.name++ # Edge deduplication: combination of subject-predicate-object triple+ edge_key_fn = lambda x: f"{x.startNode.name}|{x.name}|{x.endNode.name}"++ # 2. Edge consistency check: tell AutoGraph which nodes this edge connects+ nodes_in_edge_fn = lambda x: (x.startNode.name, x.endNode.name)++ # 3. Call parent class initialization+ logger.info("🚀 Initializing Atom")+ super().__init__(+ node_schema=NodeSchema,+ edge_schema=EdgeSchema,+ node_key_extractor=node_key_fn,+ edge_key_extractor=edge_key_fn,+ nodes_in_edge_extractor=nodes_in_edge_fn,+ llm_client=llm_client,+ embedder=embedder,+ # Enforce one-stage extraction (first extract edges, then derive nodes)+ extraction_mode="one_stage",+ # Inject customized prompts+ prompt_for_edge_extraction=Atom_EDGE_EXTRACTION_PROMPT,+ # Configure deduplication strategy+ node_strategy_or_merger=MergeStrategy.KEEP_EXISTING,+ edge_strategy_or_merger=MergeStrategy.KEEP_EXISTING,+ # Optimize indexing: only index name field+ node_fields_for_index=["name", "label"],+ edge_fields_for_index=["startNode", "name", "endNode"],+ # Other parameters+ chunk_size=chunk_size,+ chunk_overlap=chunk_overlap,+ max_workers=max_workers,+ verbose=verbose,+ )++ # ==================== Extraction Pipeline ====================++ def _extract_data(self, text: str) -> AutoGraphSchema:+ """Extract atomic facts first, then extract edges (Two-stage extraction).++ Process:+ 1. Split text into chunks.+ 2. Batch extract atomic facts from chunks.+ 3. Consolidate facts into a unified context.+ 4. Split consolidated facts into chunks.+ 5. Batch extract edges from fact chunks.+ 6. Post-process: Set t_obs timestamp & Derive nodes.+ 7. Merge all partial graphs into one global graph.++ Args:+ text: Input text.++ Returns:+ Extracted and validated graph.+ """+ obs_date_str = self.observation_date or datetime.now().strftime("%Y-%m-%d")++ # ==================== Step 1: Extract Atomic Facts ====================+ logger.info("🔍 [Phase 1] Extracting Atomic Facts...")++ # 1. Prepare raw chunks+ if len(text) <= self.chunk_size:+ raw_chunks = [text]+ else:+ raw_chunks = self.text_splitter.split_text(text)++ # 2. Define Fact Extraction Chain+ fact_prompt_template = ChatPromptTemplate.from_template(+ f"{Atom_FACTOID_EXTRACTION_PROMPT}\n\nObservation Date: {{observation_date}}\n\n### Source Text:\n{{chunk_text}}"+ )+ fact_chain = fact_prompt_template | self.llm_client.with_structured_output(+ AtomicFactSchema+ )++ # 3. Batch Extract Facts+ fact_inputs = [+ {"chunk_text": chunk, "observation_date": obs_date_str}+ for chunk in raw_chunks+ ]+ chunk_fact_lists = fact_chain.batch(+ fact_inputs, config={"max_concurrency": self.max_workers}+ )++ # 4. Consolidate Facts+ all_facts = []+ for fact_list in chunk_fact_lists:+ if fact_list and fact_list.atomic_fact:+ all_facts.extend(fact_list.atomic_fact)++ logger.info(f"✅ Extracted {len(all_facts)} atomic facts.")++ if not all_facts:+ logger.warning("⚠️ No facts extracted. Returning empty graph.")+ return self._create_empty_instance().data++ # ==================== Step 2: Extract Edges from Facts ====================+ logger.info("🔍 [Phase 2] Extracting Edges from Atomic Facts...")++ # 5. Create new context from facts (join with newlines)+ fact_context_text = "\n".join(all_facts)++ # 6. Split Fact Context into chunks+ if len(fact_context_text) <= self.chunk_size:+ fact_chunks = [fact_context_text]+ else:+ fact_chunks = self.text_splitter.split_text(fact_context_text)++ # 7. Batch Extract Edges directly from Fact Chunks+ # Format the edge prompt with observation date+ edge_prompt_with_date = self.edge_prompt.format(obs_timestamp=obs_date_str)+ edge_prompt_template = ChatPromptTemplate.from_template(+ f"{edge_prompt_with_date}\n\n### Source Text:\n{{chunk_text}}"+ )+ edge_chain = edge_prompt_template | self.llm_client.with_structured_output(+ self.edge_list_schema+ )++ edge_inputs = [{"chunk_text": chunk} for chunk in fact_chunks]+ chunk_edge_lists = edge_chain.batch(+ edge_inputs, config={"max_concurrency": self.max_workers}+ )++ # ==================== Step 3: Post-process & Merge ====================++ # 8. Post-process: Derive Nodes & Set t_obs+ all_nodes_lists, all_edges_lists = [], []++ for edge_list in chunk_edge_lists:+ derived_nodes, processed_edges = [], []++ if edge_list and edge_list.items:+ for edge in edge_list.items:+ edge: EdgeSchema+ # Set t_obs to extraction timestamp+ edge.t_obs = [obs_date_str]++ # Derive nodes from edges+ derived_nodes.append(edge.startNode)+ derived_nodes.append(edge.endNode)+ processed_edges.append(edge)++ all_nodes_lists.append(derived_nodes)+ all_edges_lists.append(processed_edges)++ # 9. Construct Partial Graphs+ partial_graphs = (all_nodes_lists, all_edges_lists)++ # 10. Global Merge+ raw_graph = self.merge_batch_data(partial_graphs)++ return self._prune_dangling_edges(raw_graph)++ # ==================== Node Deduplication ====================++ def match_nodes_and_update_edges(self, threshold: float = 0.8) -> "Atom":+ """Match nodes in the graph and update edges accordingly using SemHash with embeddings.++ This method identifies and merges similar nodes based on semantic similarity+ using embeddings from the instance's embedder. It updates edges to reflect+ any changes in node identities.++ Args:+ threshold: Similarity threshold for matching nodes (0.0 to 1.0). Defaults to 0.8.++ Returns:+ The updated Atom instance with matched nodes and updated edges.+ """+ logger.info(+ f"🔄 Starting node matching and edge update (threshold={threshold})..."+ )++ nodes, edges = self.nodes, self.edges++ if not nodes:+ logger.warning("⚠️ No nodes to match; skipping matching.")+ return self++ # 1. Generate Embeddings using self.embedder.embed_documents+ node_names = [n.name for n in nodes]+ logger.info(f"📊 Generating embeddings for {len(node_names)} nodes...")++ try:+ embeddings = self.embedder.embed_documents(node_names)+ except Exception as e:+ logger.error(f"❌ Failed to generate embeddings: {e}")+ return self++ # 2. Initialize SemHash from embeddings+ semhash = SemHash.from_embeddings(+ embeddings=embeddings,+ records=node_names,+ model=None, # Embeddings already provided+ )++ # 3. Run Deduplication+ result = semhash.self_deduplicate(threshold=threshold)++ # 4. Build Mapping from duplicates result+ mapping = {} # old_name -> new_name+ for record in result.filtered:+ if record.duplicates and len(record.duplicates) > 0:+ # Map the record to the first duplicate (representative)+ mapping[record.record] = record.duplicates[0][0]++ if not mapping:+ logger.info("✅ No similar nodes found above threshold.")+ return self++ logger.info(f"🔗 Found {len(mapping)} nodes to merge.")++ # 5. Apply mapping to graph data+ # Update Node names+ for node in nodes:+ if node.name in mapping:+ node.name = mapping[node.name]++ # Update Edge references (startNode and endNode are nested NodeSchema objects)+ for edge in edges:+ if edge.startNode.name in mapping:+ edge.startNode.name = mapping[edge.startNode.name]+ if edge.endNode.name in mapping:+ edge.endNode.name = mapping[edge.endNode.name]++ # 6. Update internal state+ new_data = AutoGraphSchema(nodes=nodes, edges=edges)+ self._set_data_state(new_data)++ logger.info(+ f"✅ Node matching complete: Nodes {len(nodes)} -> {len(self.nodes)}, Edges: {len(edges)}"+ )+ return selfdiff --git a/hyperextract/graphs/itext2kg_star.py b/hyperextract/graphs/itext2kg_star.pyindex c2af9ae..4b75914 100644--- a/hyperextract/graphs/itext2kg_star.py+++ b/hyperextract/graphs/itext2kg_star.py@@ -7,7 +7,7 @@ standardized triple-based knowledge graphs.Prompts and schemas are adapted from the original iText2KG_Star implementation."""-from typing import Optional, List+from typing import Optionalfrom datetime import datetimefrom pydantic import BaseModel, Fieldfrom semhash import SemHash@@ -16,7 +16,7 @@ from langchain_core.prompts import ChatPromptTemplatefrom langchain_core.embeddings import Embeddingsfrom ontomem.merger import MergeStrategy-from .base import AutoGraph, AutoGraphSchema, NodeListSchema, EdgeListSchema+from .base import AutoGraph, AutoGraphSchemafrom ..utils.logging import logger# ==============================================================================@@ -144,7 +144,7 @@ class iText2KG_Star(AutoGraph[NodeSchema, EdgeSchema]):Args:llm_client: Language model for extractionembedder: Embedding model for vector indexing- observation_date: Date when the extraction was performed, like '1997-10-10' or '1997-10-10 23:59:59'.+ observation_date: Date when the extraction was performed, like '1997-10-10' or '1997-10-10 23:59:59'.If None, uses current date and time.chunk_size: Characters per chunkchunk_overlap: Overlapping characters between chunks