Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

Refactor schema definitions and remove merger utilities

yifanfeng975ecc00f75ca9
diff --git a/hyperextract/templates/en/business/ecosystem.py b/hyperextract/templates/en/business/ecosystem.py
index ae0fd99..150a350 100644
--- a/hyperextract/templates/en/business/ecosystem.py
+++ b/hyperextract/templates/en/business/ecosystem.py
@@ -8,22 +8,42 @@ from hyperextract.types import AutoHypergraph
# 1. Schema Definitions
# ==============================================================================
+
class IndustryParticipant(BaseModel):
"""
An entity (Company, Regulatory Body, NGO) within an industrial ecosystem.
"""
- name: str = Field(description="Name of the actor (e.g., 'TSMC', 'European Commission').")
- rank: Optional[str] = Field(None, description="Market position: 'Tier 1 Supplier', 'Market Leader', 'Challenger'.")
- technological_focus: Optional[str] = Field(None, description="Primary tech or domain (e.g., '5G', 'Lithography').")
+
+ name: str = Field(
+ description="Name of the actor (e.g., 'TSMC', 'European Commission')."
+ )
+ rank: Optional[str] = Field(
+ None,
+ description="Market position: 'Tier 1 Supplier', 'Market Leader', 'Challenger'.",
+ )
+ technological_focus: Optional[str] = Field(
+ None, description="Primary tech or domain (e.g., '5G', 'Lithography')."
+ )
+
class CollectiveAlliance(BaseModel):
"""
A hyperedge representing a consortium, alliance, market cartel, or joint standard-setting body.
"""
- alliance_id: str = Field(description="Name or identifier of the alliance (e.g., 'CHIPS Alliance').")
- participants: List[str] = Field(description="Names of all entities collaborating or competing in this group.")
- shared_goal: str = Field(description="The objective: 'Standardization', 'Anti-monopoly Review', 'Joint R&D'.")
- governance: Optional[str] = Field(None, description="Rules of participation or leading entity.")
+
+ alliance_id: str = Field(
+ description="Name or identifier of the alliance (e.g., 'CHIPS Alliance')."
+ )
+ participants: List[str] = Field(
+ description="Names of all entities collaborating or competing in this group."
+ )
+ shared_goal: str = Field(
+ description="The objective: 'Standardization', 'Anti-monopoly Review', 'Joint R&D'."
+ )
+ governance: Optional[str] = Field(
+ None, description="Rules of participation or leading entity."
+ )
+
# ==============================================================================
# 2. Prompts
@@ -59,10 +79,12 @@ ECOSYSTEM_EDGE_PROMPT = (
# 3. Template Class
# ==============================================================================
+
class MarketEcosystemHyper(AutoHypergraph[IndustryParticipant, CollectiveAlliance]):
"""
Hypergraph template for industrial clusters, ecosystem coalitions, and complex market alliances.
"""
+
def __init__(
self,
llm_client: BaseChatModel,
@@ -73,7 +95,7 @@ class MarketEcosystemHyper(AutoHypergraph[IndustryParticipant, CollectiveAllianc
chunk_overlap: int = 256,
max_workers: int = 10,
verbose: bool = False,
- **kwargs: Any
+ **kwargs: Any,
):
super().__init__(
node_schema=IndustryParticipant,
@@ -91,5 +113,5 @@ class MarketEcosystemHyper(AutoHypergraph[IndustryParticipant, CollectiveAllianc
prompt=ECOSYSTEM_CONSOLIDATED_PROMPT,
prompt_for_node_extraction=ECOSYSTEM_NODE_PROMPT,
prompt_for_edge_extraction=ECOSYSTEM_EDGE_PROMPT,
- **kwargs
+ **kwargs,
)
diff --git a/hyperextract/templates/en/business/equity.py b/hyperextract/templates/en/business/equity.py
index d1b24e8..930c014 100644
--- a/hyperextract/templates/en/business/equity.py
+++ b/hyperextract/templates/en/business/equity.py
@@ -8,24 +8,47 @@ from hyperextract.types import AutoGraph
# 1. Schema Definitions
# ==============================================================================
+
class StakeholderNode(BaseModel):
"""
A legal or natural person holding equity or control in a business.
"""
- name: str = Field(description="Full registry name of the company or individual's full name.")
- type: str = Field(description="Type: 'Natural Person', 'Public Company', 'Private Company', 'State-Owned Enterprise', 'Venture Capital'.")
- jurisdiction: Optional[str] = Field(None, description="Country or region of incorporation/residence.")
- is_ubo: bool = Field(False, description="Whether this entity is identified as an Ultimate Beneficial Owner.")
+
+ name: str = Field(
+ description="Full registry name of the company or individual's full name."
+ )
+ type: str = Field(
+ description="Type: 'Natural Person', 'Public Company', 'Private Company', 'State-Owned Enterprise', 'Venture Capital'."
+ )
+ jurisdiction: Optional[str] = Field(
+ None, description="Country or region of incorporation/residence."
+ )
+ is_ubo: bool = Field(
+ False,
+ description="Whether this entity is identified as an Ultimate Beneficial Owner.",
+ )
+
class OwnershipRelation(BaseModel):
"""
A stakeholding or control link between two entities.
"""
+
source: str = Field(description="The shareholder or parent entity.")
- target: str = Field(description="The company being owned or controlled (subsidiary).")
- stake_percentage: Optional[float] = Field(None, description="Percentage of ownership (0.0 to 100.0).")
- control_type: str = Field("Equity", description="Nature of link: 'Equity Acquisition', 'Voting Rights', 'Nominee Shareholding', 'Direct Control'.")
- as_of_date: Optional[str] = Field(None, description="The date or fiscal period when this ownership was recorded.")
+ target: str = Field(
+ description="The company being owned or controlled (subsidiary)."
+ )
+ stake_percentage: Optional[float] = Field(
+ None, description="Percentage of ownership (0.0 to 100.0)."
+ )
+ control_type: str = Field(
+ "Equity",
+ description="Nature of link: 'Equity Acquisition', 'Voting Rights', 'Nominee Shareholding', 'Direct Control'.",
+ )
+ as_of_date: Optional[str] = Field(
+ None, description="The date or fiscal period when this ownership was recorded."
+ )
+
# ==============================================================================
# 2. Prompts
@@ -61,10 +84,12 @@ EQUITY_EDGE_PROMPT = (
# 3. Template Class
# ==============================================================================
+
class EquityStructureGraph(AutoGraph[StakeholderNode, OwnershipRelation]):
"""
Precision template for corporate registries, M&A analysis, and KYC/UBO tracking.
"""
+
def __init__(
self,
llm_client: BaseChatModel,
@@ -75,13 +100,15 @@ class EquityStructureGraph(AutoGraph[StakeholderNode, OwnershipRelation]):
chunk_overlap: int = 256,
max_workers: int = 10,
verbose: bool = False,
- **kwargs: Any
+ **kwargs: Any,
):
super().__init__(
node_schema=StakeholderNode,
edge_schema=OwnershipRelation,
node_key_extractor=lambda x: x.name.strip(),
- edge_key_extractor=lambda x: f"{x.source.strip()}->{x.target.strip()}_at_{x.stake_percentage}%",
+ edge_key_extractor=lambda x: (
+ f"{x.source.strip()}->{x.target.strip()}_at_{x.stake_percentage}%"
+ ),
nodes_in_edge_extractor=lambda x: (x.source.strip(), x.target.strip()),
llm_client=llm_client,
embedder=embedder,
@@ -93,5 +120,5 @@ class EquityStructureGraph(AutoGraph[StakeholderNode, OwnershipRelation]):
prompt=EQUITY_CONSOLIDATED_PROMPT,
prompt_for_node_extraction=EQUITY_NODE_PROMPT,
prompt_for_edge_extraction=EQUITY_EDGE_PROMPT,
- **kwargs
+ **kwargs,
)
diff --git a/hyperextract/templates/en/business/strategy.py b/hyperextract/templates/en/business/strategy.py
index 3398327..c4988b2 100644
--- a/hyperextract/templates/en/business/strategy.py
+++ b/hyperextract/templates/en/business/strategy.py
@@ -8,24 +8,44 @@ from hyperextract.types import AutoTemporalGraph
# 1. Schema Definitions
# ==============================================================================
+
class StrategicNode(BaseModel):
"""
An organizational entity, business division, or strategic objective.
"""
- name: str = Field(description="Name of the strategic unit or goal (e.g., 'Azure', 'Carbon Neutrality').")
- scope: str = Field(description="Scope: 'Global', 'Regional', 'Product-specific', 'Corporate-wide'.")
- importance: Optional[str] = Field(None, description="Criticality: 'Core Business', 'Emerging Market', 'Legacy'.")
+
+ name: str = Field(
+ description="Name of the strategic unit or goal (e.g., 'Azure', 'Carbon Neutrality')."
+ )
+ scope: str = Field(
+ description="Scope: 'Global', 'Regional', 'Product-specific', 'Corporate-wide'."
+ )
+ importance: Optional[str] = Field(
+ None, description="Criticality: 'Core Business', 'Emerging Market', 'Legacy'."
+ )
+
class StrategicPivot(BaseModel):
"""
A temporal transition representing a change in strategic focus, investment, or divestiture.
"""
+
source: str = Field(description="The original area of focus or acting entity.")
target: str = Field(description="The new target, direction, or goal.")
- action: str = Field(description="Strategic action: 'Pivot to', 'Acquire', 'Divest', 'Merge', 'Sunset', 'Scale'.")
- timestamp: str = Field(description="When the shift occurred (e.g., 'Q4 2022', '2025-01-01').")
- rationale: Optional[str] = Field(None, description="The 'Why' behind the move (market pressure, CEO change, tech shift).")
- budget: Optional[str] = Field(None, description="Financial commitment mentioned alongside the pivot.")
+ action: str = Field(
+ description="Strategic action: 'Pivot to', 'Acquire', 'Divest', 'Merge', 'Sunset', 'Scale'."
+ )
+ timestamp: str = Field(
+ description="When the shift occurred (e.g., 'Q4 2022', '2025-01-01')."
+ )
+ rationale: Optional[str] = Field(
+ None,
+ description="The 'Why' behind the move (market pressure, CEO change, tech shift).",
+ )
+ budget: Optional[str] = Field(
+ None, description="Financial commitment mentioned alongside the pivot."
+ )
+
# ==============================================================================
# 2. Prompts
@@ -61,10 +81,12 @@ STRATEGY_EDGE_PROMPT = (
# 3. Template Class
# ==============================================================================
+
class StrategicChainGraph(AutoTemporalGraph[StrategicNode, StrategicPivot]):
"""
Temporal template for tracking corporate strategy, longitudinal organizational change, and market shifts.
"""
+
def __init__(
self,
llm_client: BaseChatModel,
@@ -75,13 +97,15 @@ class StrategicChainGraph(AutoTemporalGraph[StrategicNode, StrategicPivot]):
chunk_overlap: int = 256,
max_workers: int = 10,
verbose: bool = False,
- **kwargs: Any
+ **kwargs: Any,
):
super().__init__(
node_schema=StrategicNode,
edge_schema=StrategicPivot,
node_key_extractor=lambda x: x.name.strip(),
- edge_key_extractor=lambda x: f"{x.source.strip()}-({x.action})->{x.target.strip()}",
+ edge_key_extractor=lambda x: (
+ f"{x.source.strip()}-({x.action})->{x.target.strip()}"
+ ),
time_in_edge_extractor=lambda x: x.timestamp.strip(),
nodes_in_edge_extractor=lambda x: (x.source.strip(), x.target.strip()),
llm_client=llm_client,
@@ -94,5 +118,5 @@ class StrategicChainGraph(AutoTemporalGraph[StrategicNode, StrategicPivot]):
prompt=STRATEGY_CONSOLIDATED_PROMPT,
prompt_for_node_extraction=STRATEGY_NODE_PROMPT,
prompt_for_edge_extraction=STRATEGY_EDGE_PROMPT,
- **kwargs
+ **kwargs,
)
diff --git a/hyperextract/templates/en/chemistry/molecular.py b/hyperextract/templates/en/chemistry/molecular.py
index 7117d3b..370e3a1 100644
--- a/hyperextract/templates/en/chemistry/molecular.py
+++ b/hyperextract/templates/en/chemistry/molecular.py
@@ -8,24 +8,41 @@ from hyperextract.types import AutoSpatialGraph
# 1. Schema Definitions
# ==============================================================================
+
class AtomNode(BaseModel):
"""
An individual atom in a chemical structure.
"""
- name: str = Field(description="Atom identifier (e.g., 'C1', 'O2') or element symbol.")
+
+ name: str = Field(
+ description="Atom identifier (e.g., 'C1', 'O2') or element symbol."
+ )
element: str = Field(description="The chemical element (e.g., 'Carbon', 'H').")
- hybridization: Optional[str] = Field(None, description="Electronic state: 'sp3', 'sp2', 'sp'.")
- formal_charge: int = Field(0, description="The electrical charge assigned to the atom.")
+ hybridization: Optional[str] = Field(
+ None, description="Electronic state: 'sp3', 'sp2', 'sp'."
+ )
+ formal_charge: int = Field(
+ 0, description="The electrical charge assigned to the atom."
+ )
+
class ChemicalBond(BaseModel):
"""
A chemical bond between two atoms, including spatial/positional metadata.
"""
+
source: str = Field(description="The originating atom (e.g., 'C1').")
target: str = Field(description="The connected atom (e.g., 'C2').")
- bond_type: str = Field(description="Type: 'Single', 'Double', 'Triple', 'Aromatic', 'Hydrogen'.")
- spatial_position: str = Field(description="The physical or logical position in the molecule (e.g., 'C4 position', 'Axial', 'Equatorial').")
- bond_length: Optional[float] = Field(None, description="Distance between atoms in Angstroms.")
+ bond_type: str = Field(
+ description="Type: 'Single', 'Double', 'Triple', 'Aromatic', 'Hydrogen'."
+ )
+ spatial_position: str = Field(
+ description="The physical or logical position in the molecule (e.g., 'C4 position', 'Axial', 'Equatorial')."
+ )
+ bond_length: Optional[float] = Field(
+ None, description="Distance between atoms in Angstroms."
+ )
+
# ==============================================================================
# 2. Prompts
@@ -61,10 +78,12 @@ MOLECULAR_EDGE_PROMPT = (
# 3. Template Class
# ==============================================================================
+
class MolecularStructureGraph(AutoSpatialGraph[AtomNode, ChemicalBond]):
"""
Spatial graph template for high-resolution molecular modeling and structural biology.
"""
+
def __init__(
self,
llm_client: BaseChatModel,
@@ -75,13 +94,15 @@ class MolecularStructureGraph(AutoSpatialGraph[AtomNode, ChemicalBond]):
chunk_overlap: int = 256,
max_workers: int = 10,
verbose: bool = False,
- **kwargs: Any
+ **kwargs: Any,
):
super().__init__(
node_schema=AtomNode,
edge_schema=ChemicalBond,
node_key_extractor=lambda x: x.name.strip(),
- edge_key_extractor=lambda x: f"{x.source.strip()}--({x.bond_type})-->{x.target.strip()}",
+ edge_key_extractor=lambda x: (
+ f"{x.source.strip()}--({x.bond_type})-->{x.target.strip()}"
+ ),
location_in_edge_extractor=lambda x: x.spatial_position.strip(),
nodes_in_edge_extractor=lambda x: (x.source.strip(), x.target.strip()),
llm_client=llm_client,
@@ -94,5 +115,5 @@ class MolecularStructureGraph(AutoSpatialGraph[AtomNode, ChemicalBond]):
prompt=MOLECULAR_CONSOLIDATED_PROMPT,
prompt_for_node_extraction=MOLECULAR_NODE_PROMPT,
prompt_for_edge_extraction=MOLECULAR_EDGE_PROMPT,
- **kwargs
+ **kwargs,
)
diff --git a/hyperextract/templates/en/chemistry/protocol.py b/hyperextract/templates/en/chemistry/protocol.py
index 85185fc..5b657fc 100644
--- a/hyperextract/templates/en/chemistry/protocol.py
+++ b/hyperextract/templates/en/chemistry/protocol.py
@@ -8,25 +8,44 @@ from hyperextract.types import AutoTemporalGraph
# 1. Schema Definitions
# ==============================================================================
+
class LabOperation(BaseModel):
"""
A specific step or action in a chemical or biological protocol (e.g., 'Centrifugation', 'Titration').
"""
+
op_id: str = Field(description="Action name (e.g., 'Centrifuge', 'Heat', 'Add').")
- equipment: Optional[str] = Field(None, description="Device used (e.g., 'Microcentrifuge', 'Bunsen burner').")
- reagents_added: List[str] = Field(default_factory=list, description="Chemicals introduced during this specific step.")
- precautions: Optional[str] = Field(None, description="Safety or accuracy warnings (e.g., 'Keep on ice').")
+ equipment: Optional[str] = Field(
+ None, description="Device used (e.g., 'Microcentrifuge', 'Bunsen burner')."
+ )
+ reagents_added: List[str] = Field(
+ default_factory=list,
+ description="Chemicals introduced during this specific step.",
+ )
+ precautions: Optional[str] = Field(
+ None, description="Safety or accuracy warnings (e.g., 'Keep on ice')."
+ )
+
class ProtocolTransition(BaseModel):
"""
The sequential transition between protocol steps, capturing duration and order.
"""
+
transition_id: str = Field(description="Unique id for the step transition.")
source_step: str = Field(description="The preceding operation.")
target_step: str = Field(description="The succeeding operation.")
- duration: Optional[str] = Field(None, description="Time spent in or between steps (e.g., '15 mins', 'Overnight').")
- timestamp: str = Field(description="The sequence order or relative time marker (e.g., 'T+0', 'Step 2').")
- wait_condition: Optional[str] = Field(None, description="Condition to proceed (e.g., 'Until precipitate forms').")
+ duration: Optional[str] = Field(
+ None,
+ description="Time spent in or between steps (e.g., '15 mins', 'Overnight').",
+ )
+ timestamp: str = Field(
+ description="The sequence order or relative time marker (e.g., 'T+0', 'Step 2')."
+ )
+ wait_condition: Optional[str] = Field(
+ None, description="Condition to proceed (e.g., 'Until precipitate forms')."
+ )
+
# ==============================================================================
# 2. Prompts
@@ -63,10 +82,12 @@ PROTOCOL_EDGE_PROMPT = (
# 3. Template Class
# ==============================================================================
+
class LabProtocolTemporal(AutoTemporalGraph[LabOperation, ProtocolTransition]):
"""
Temporal graph template for capturing step-by-step laboratory procedures and experimental workflows.
"""
+
def __init__(
self,
llm_client: BaseChatModel,
@@ -77,14 +98,17 @@ class LabProtocolTemporal(AutoTemporalGraph[LabOperation, ProtocolTransition]):
chunk_overlap: int = 256,
max_workers: int = 10,
verbose: bool = False,
- **kwargs: Any
+ **kwargs: Any,
):
super().__init__(
node_schema=LabOperation,
edge_schema=ProtocolTransition,
node_key_extractor=lambda x: x.op_id.strip(),
edge_key_extractor=lambda x: x.transition_id.strip(),
- nodes_in_edge_extractor=lambda x: (x.source_step.strip(), x.target_step.strip()),
+ nodes_in_edge_extractor=lambda x: (
+ x.source_step.strip(),
+ x.target_step.strip(),
+ ),
time_in_edge_extractor=lambda x: x.timestamp.strip(),
llm_client=llm_client,
embedder=embedder,
@@ -96,5 +120,5 @@ class LabProtocolTemporal(AutoTemporalGraph[LabOperation, ProtocolTransition]):
prompt=PROTOCOL_CONSOLIDATED_PROMPT,
prompt_for_node_extraction=PROTOCOL_NODE_PROMPT,
prompt_for_edge_extraction=PROTOCOL_EDGE_PROMPT,
- **kwargs
+ **kwargs,
)
diff --git a/hyperextract/templates/en/chemistry/reaction.py b/hyperextract/templates/en/chemistry/reaction.py
index 4c2e956..2da5829 100644
--- a/hyperextract/templates/en/chemistry/reaction.py
+++ b/hyperextract/templates/en/chemistry/reaction.py
@@ -8,27 +8,49 @@ from hyperextract.types import AutoHypergraph
# 1. Schema Definitions
# ==============================================================================
+
class ChemicalEntity(BaseModel):
"""
A chemical substance (Reagent, Product, Catalyst, Solvent).
"""
+
name: str = Field(description="IUPAC name or common name (e.g., 'Sulfuric Acid').")
- formula: Optional[str] = Field(None, description="Chemical formula (e.g., 'H2SO4').")
+ formula: Optional[str] = Field(
+ None, description="Chemical formula (e.g., 'H2SO4')."
+ )
cas_number: Optional[str] = Field(None, description="CAS Registry Number.")
- phase: str = Field("Unknown", description="Physical state: 'Gas', 'Liquid', 'Solid', 'Aqueous'.")
+ phase: str = Field(
+ "Unknown", description="Physical state: 'Gas', 'Liquid', 'Solid', 'Aqueous'."
+ )
+
class ReactionEvent(BaseModel):
"""
A chemical reaction involving multiple reagents, specific conditions, and resulting products.
"""
- reaction_id: str = Field(description="Name or short description of the reaction (e.g., 'Haber Process').")
+
+ reaction_id: str = Field(
+ description="Name or short description of the reaction (e.g., 'Haber Process')."
+ )
reactants: List[str] = Field(description="List of starting chemical entity names.")
products: List[str] = Field(description="List of resulting chemical entity names.")
- catalysts: List[str] = Field(default_factory=list, description="Names of substances facilitating the reaction.")
- solvents: List[str] = Field(default_factory=list, description="The medium in which the reaction occurs.")
- temperature: Optional[str] = Field(None, description="Operating temperature (e.g., '250°C').")
- pressure: Optional[str] = Field(None, description="Operating pressure (e.g., '200 atm').")
- yield_info: Optional[str] = Field(None, description="Percentage/amount of product obtained.")
+ catalysts: List[str] = Field(
+ default_factory=list,
+ description="Names of substances facilitating the reaction.",
+ )
+ solvents: List[str] = Field(
+ default_factory=list, description="The medium in which the reaction occurs."
+ )
+ temperature: Optional[str] = Field(
+ None, description="Operating temperature (e.g., '250°C')."
+ )
+ pressure: Optional[str] = Field(
+ None, description="Operating pressure (e.g., '200 atm')."
+ )
+ yield_info: Optional[str] = Field(
+ None, description="Percentage/amount of product obtained."
+ )
+
# ==============================================================================
# 2. Prompts
@@ -64,10 +86,12 @@ REACTION_EDGE_PROMPT = (
# 3. Template Class
# ==============================================================================
+
class ChemicalReactionHyper(AutoHypergraph[ChemicalEntity, ReactionEvent]):
"""
Hypergraph template for complex chemical synthesis, industrial processes, and reaction pathways.
"""
+
def __init__(
self,
llm_client: BaseChatModel,
@@ -78,14 +102,16 @@ class ChemicalReactionHyper(AutoHypergraph[ChemicalEntity, ReactionEvent]):
chunk_overlap: int = 256,
max_workers: int = 10,
verbose: bool = False,
- **kwargs: Any
+ **kwargs: Any,
):
super().__init__(
node_schema=ChemicalEntity,
edge_schema=ReactionEvent,
node_key_extractor=lambda x: x.name.strip(),
edge_key_extractor=lambda x: x.reaction_id.strip(),
- nodes_in_edge_extractor=lambda x: tuple(set(x.reactants + x.products + x.catalysts + x.solvents)),
+ nodes_in_edge_extractor=lambda x: tuple(
+ set(x.reactants + x.products + x.catalysts + x.solvents)
+ ),
llm_client=llm_client,
embedder=embedder,
extraction_mode=extraction_mode,
@@ -96,5 +122,5 @@ class ChemicalReactionHyper(AutoHypergraph[ChemicalEntity, ReactionEvent]):
prompt=REACTION_CONSOLIDATED_PROMPT,
prompt_for_node_extraction=REACTION_NODE_PROMPT,
prompt_for_edge_extraction=REACTION_EDGE_PROMPT,
- **kwargs
+ **kwargs,
)
diff --git a/hyperextract/templates/zh/business/ecosystem.py b/hyperextract/templates/zh/business/ecosystem.py
index 1a3375c..5b0c32c 100644
--- a/hyperextract/templates/zh/business/ecosystem.py
+++ b/hyperextract/templates/zh/business/ecosystem.py
@@ -8,23 +8,34 @@ from hyperextract.types import AutoHypergraph
# 1. Schema Definitions
# ==============================================================================
+
class IndustryParticipant(BaseModel):
"""
产业生态系统中的实体(公司、监管机构、非政府组织)。
"""
+
name: str = Field(description="参与者名称(如:'台积电'、'欧盟委员会')。")
- rank: Optional[str] = Field(None, description="市场地位:'一级供应商'、'市场领导者'、'挑战者'。")
- technological_focus: Optional[str] = Field(None, description="主要技术或领域(如:'5G'、'光刻技术')。")
+ rank: Optional[str] = Field(
+ None, description="市场地位:'一级供应商'、'市场领导者'、'挑战者'。"
+ )
+ technological_focus: Optional[str] = Field(
+ None, description="主要技术或领域(如:'5G'、'光刻技术')。"
+ )
+
class CollectiveAlliance(BaseModel):
"""
代表财团、联盟、市场卡特尔或联合标准制定机构的超边。
"""
+
alliance_id: str = Field(description="联盟的名称或标识符(如:'芯片联盟')。")
- participants: List[str] = Field(description="在该团体中进行合作或竞争的所有实体的名称列表。")
+ participants: List[str] = Field(
+ description="在该团体中进行合作或竞争的所有实体的名称列表。"
+ )
shared_goal: str = Field(description="目标:'标准化'、'反垄断审查'、'联合研发'。")
governance: Optional[str] = Field(None, description="参与原则或领导实体。")
+
# ==============================================================================
# 2. Prompts
# ==============================================================================
@@ -59,10 +70,12 @@ ECOSYSTEM_EDGE_PROMPT = (
# 3. Template Class
# ==============================================================================
+
class MarketEcosystemHyper(AutoHypergraph[IndustryParticipant, CollectiveAlliance]):
"""
用于产业集群、生态系统联盟和复杂市场协同的超图模板。
"""
+
def __init__(
self,
llm_client: BaseChatModel,
@@ -73,7 +86,7 @@ class MarketEcosystemHyper(AutoHypergraph[IndustryParticipant, CollectiveAllianc
chunk_overlap: int = 256,
max_workers: int = 10,
verbose: bool = False,
- **kwargs: Any
+ **kwargs: Any,
):
super().__init__(
node_schema=IndustryParticipant,
@@ -91,5 +104,5 @@ class MarketEcosystemHyper(AutoHypergraph[IndustryParticipant, CollectiveAllianc
prompt=ECOSYSTEM_CONSOLIDATED_PROMPT,
prompt_for_node_extraction=ECOSYSTEM_NODE_PROMPT,
prompt_for_edge_extraction=ECOSYSTEM_EDGE_PROMPT,
- **kwargs
+ **kwargs,
)
diff --git a/hyperextract/templates/zh/business/equity.py b/hyperextract/templates/zh/business/equity.py
index ea3973e..8e4c68f 100644
--- a/hyperextract/templates/zh/business/equity.py
+++ b/hyperextract/templates/zh/business/equity.py
@@ -8,24 +8,39 @@ from hyperextract.types import AutoGraph
# 1. Schema Definitions
# ==============================================================================
+
class StakeholderNode(BaseModel):
"""
在业务中持有股份或拥有控制权的法人或自然人。
"""
+
name: str = Field(description="公司注册全称或个人全名。")
- type: str = Field(description="类型:'自然人'、'上市公司'、'私有公司'、'国有企业'、'风险投资'。")
- jurisdiction: Optional[str] = Field(None, description="注册地或居住地所属的国家或地区。")
+ type: str = Field(
+ description="类型:'自然人'、'上市公司'、'私有公司'、'国有企业'、'风险投资'。"
+ )
+ jurisdiction: Optional[str] = Field(
+ None, description="注册地或居住地所属的国家或地区。"
+ )
is_ubo: bool = Field(False, description="该实体是否被识别为最终受益人(UBO)。")
+
class OwnershipRelation(BaseModel):
"""
两个实体之间的持股或控制联系。
"""
+
source: str = Field(description="股东或母公司实体。")
target: str = Field(description="被拥有或控制的公司(子公司)。")
- stake_percentage: Optional[float] = Field(None, description="持股比例(0.0 到 100.0)。")
- control_type: str = Field("Equity", description="联系性质:'股权收购'、'投票权'、'代持股份'、'直接控制'。")
- as_of_date: Optional[str] = Field(None, description="记录该所有权的时间点或财务周期。")
+ stake_percentage: Optional[float] = Field(
+ None, description="持股比例(0.0 到 100.0)。"
+ )
+ control_type: str = Field(
+ "Equity", description="联系性质:'股权收购'、'投票权'、'代持股份'、'直接控制'。"
+ )
+ as_of_date: Optional[str] = Field(
+ None, description="记录该所有权的时间点或财务周期。"
+ )
+
# ==============================================================================
# 2. Prompts
@@ -61,10 +76,12 @@ EQUITY_EDGE_PROMPT = (
# 3. Template Class
# ==============================================================================
+
class EquityStructureGraph(AutoGraph[StakeholderNode, OwnershipRelation]):
"""
用于企业登记、并购分析及 KYC/UBO 追踪的高精度模板。
"""
+
def __init__(
self,
llm_client: BaseChatModel,
@@ -75,13 +92,15 @@ class EquityStructureGraph(AutoGraph[StakeholderNode, OwnershipRelation]):
chunk_overlap: int = 256,
max_workers: int = 10,
verbose: bool = False,
- **kwargs: Any
+ **kwargs: Any,
):
super().__init__(
node_schema=StakeholderNode,
edge_schema=OwnershipRelation,
node_key_extractor=lambda x: x.name.strip(),
- edge_key_extractor=lambda x: f"{x.source.strip()}->{x.target.strip()}_at_{x.stake_percentage}%",
+ edge_key_extractor=lambda x: (
+ f"{x.source.strip()}->{x.target.strip()}_at_{x.stake_percentage}%"
+ ),
nodes_in_edge_extractor=lambda x: (x.source.strip(), x.target.strip()),
llm_client=llm_client,
embedder=embedder,
@@ -93,5 +112,5 @@ class EquityStructureGraph(AutoGraph[StakeholderNode, OwnershipRelation]):
prompt=EQUITY_CONSOLIDATED_PROMPT,
prompt_for_node_extraction=EQUITY_NODE_PROMPT,
prompt_for_edge_extraction=EQUITY_EDGE_PROMPT,
- **kwargs
+ **kwargs,
)
diff --git a/hyperextract/templates/zh/business/strategy.py b/hyperextract/templates/zh/business/strategy.py
index 504a2c7..46d66ea 100644
--- a/hyperextract/templates/zh/business/strategy.py
+++ b/hyperextract/templates/zh/business/strategy.py
@@ -8,25 +8,38 @@ from hyperextract.types import AutoTemporalGraph
# 1. Schema Definitions
# ==============================================================================
+
class StrategicNode(BaseModel):
"""
组织实体、业务部门或战略目标。
"""
+
name: str = Field(description="战略单元或目标的名称(如:'Azure'、'碳中和')。")
scope: str = Field(description="范围:'全球'、'区域'、'特定产品'、'集团层面'。")
- importance: Optional[str] = Field(None, description="重要性:'核心业务'、'新兴市场'、'传统存量'。")
+ importance: Optional[str] = Field(
+ None, description="重要性:'核心业务'、'新兴市场'、'传统存量'。"
+ )
+
class StrategicPivot(BaseModel):
"""
代表战略重点转移、投资或剥离的时间性转变。
"""
+
source: str = Field(description="原始关注领域或施加动作的实体。")
target: str = Field(description="新的目标、方向或目的。")
- action: str = Field(description="战略动作:'转型为'、'收购'、'剥离'、'合并'、'关停'、'规模化'。")
- timestamp: str = Field(description="转型发生的时间(如:'2022年第四季度'、'2025-01-01')。")
- rationale: Optional[str] = Field(None, description="举措背后的原因(市场压力、CEO 变动、技术变革)。")
+ action: str = Field(
+ description="战略动作:'转型为'、'收购'、'剥离'、'合并'、'关停'、'规模化'。"
+ )
+ timestamp: str = Field(
+ description="转型发生的时间(如:'2022年第四季度'、'2025-01-01')。"
+ )
+ rationale: Optional[str] = Field(
+ None, description="举措背后的原因(市场压力、CEO 变动、技术变革)。"
+ )
budget: Optional[str] = Field(None, description="与转型相关的财务承诺或预算。")
+
# ==============================================================================
# 2. Prompts
# ==============================================================================
@@ -61,10 +74,12 @@ STRATEGY_EDGE_PROMPT = (
# 3. Template Class
# ==============================================================================
+
class StrategicChainGraph(AutoTemporalGraph[StrategicNode, StrategicPivot]):
"""
用于追踪公司战略、长期组织变革和市场表现的时序模板。
"""
+
def __init__(
self,
llm_client: BaseChatModel,
@@ -75,13 +90,15 @@ class StrategicChainGraph(AutoTemporalGraph[StrategicNode, StrategicPivot]):
chunk_overlap: int = 256,
max_workers: int = 10,
verbose: bool = False,
- **kwargs: Any
+ **kwargs: Any,
):
super().__init__(
node_schema=StrategicNode,
edge_schema=StrategicPivot,
node_key_extractor=lambda x: x.name.strip(),
- edge_key_extractor=lambda x: f"{x.source.strip()}-({x.action})->{x.target.strip()}",
+ edge_key_extractor=lambda x: (
+ f"{x.source.strip()}-({x.action})->{x.target.strip()}"
+ ),
time_in_edge_extractor=lambda x: x.timestamp.strip(),
nodes_in_edge_extractor=lambda x: (x.source.strip(), x.target.strip()),
llm_client=llm_client,
@@ -94,5 +111,5 @@ class StrategicChainGraph(AutoTemporalGraph[StrategicNode, StrategicPivot]):
prompt=STRATEGY_CONSOLIDATED_PROMPT,
prompt_for_node_extraction=STRATEGY_NODE_PROMPT,
prompt_for_edge_extraction=STRATEGY_EDGE_PROMPT,
- **kwargs
+ **kwargs,
)
diff --git a/hyperextract/utils/__init__.py b/hyperextract/utils/__init__.py
index 8e8a47b..5407d0f 100644
--- a/hyperextract/utils/__init__.py
+++ b/hyperextract/utils/__init__.py
@@ -1,22 +1,8 @@
"""Hyperextract utilities module."""
-from .merger import (
- BaseMerger,
- MergeStrategy,
- KeepNewMerger,
- KeepOldMerger,
- FieldMerger,
- LLMMerger,
-)
from .logging import logger, setup_logger
__all__ = [
- "BaseMerger",
- "MergeStrategy",
- "KeepNewMerger",
- "KeepOldMerger",
- "FieldMerger",
- "LLMMerger",
"logger",
"setup_logger",
]
diff --git a/hyperextract/utils/merger/__init__.py b/hyperextract/utils/merger/__init__.py
deleted file mode 100644
index 9f3ba9d..0000000
--- a/hyperextract/utils/merger/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-"""Item merger utilities for knowledge patterns.
-
-Provides a flexible merger framework with tournament-style merge algorithm
-and support for various merge strategies including LLM-powered intelligent merging.
-"""
-
-from .base import BaseMerger, MergeStrategy
-from .standard import KeepNewMerger, KeepOldMerger, FieldMerger
-from .llm import LLMMerger
-
-__all__ = [
- "BaseMerger",
- "MergeStrategy",
- "KeepNewMerger",
- "KeepOldMerger",
- "FieldMerger",
- "LLMMerger",
-]
diff --git a/hyperextract/utils/merger/base.py b/hyperextract/utils/merger/base.py
deleted file mode 100644
index aee28fd..0000000
--- a/hyperextract/utils/merger/base.py
+++ /dev/null
@@ -1,347 +0,0 @@
-"""Base merger class implementing tournament-style merge algorithm.
-
-The tournament merge algorithm efficiently handles n-way merging of duplicate items
-by organizing merges into logarithmic rounds, similar to tournament elimination.
-"""
-
-import math
-import logging
-from abc import ABC, abstractmethod
-from enum import Enum
-from typing import List, Any, Callable, TypeVar, Generic, Dict, Tuple, Optional
-from collections import defaultdict
-
-try:
- from hyperextract.utils.logging import logger
-except ImportError:
- logger = logging.getLogger(__name__)
-
-T = TypeVar("T") # Generic item type
-
-
-class MergeStrategy(str, Enum):
- """Merge strategy for handling duplicate items.
-
- This enum defines the standard strategies for merging duplicate items
- when the same unique key is encountered multiple times.
-
- Strategies:
- KEEP_OLD: Keep the existing item, discard the new one
- KEEP_NEW: Keep the new item, discard the existing one (default)
- FIELD_MERGE: Merge fields from both items (new fills old's None fields)
- LLM_MERGE: Use LLM to intelligently merge both items
-
- Example:
- >>> from hyperextract.utils.merger import MergeStrategy
- >>> strategy = MergeStrategy.FIELD_MERGE
- >>> print(strategy.value) # "field_merge"
- """
-
- KEEP_OLD = "keep_old"
- KEEP_NEW = "keep_new"
- FIELD_MERGE = "field_merge"
- LLM_MERGE = "llm_merge"
-
-
-class BaseMerger(ABC, Generic[T]):
- """Abstract base class for item mergers with tournament-style merge algorithm.
-
- This class provides the core tournament merge algorithm that efficiently handles
- n duplicate items by organizing merges into O(log n) rounds. Each round collects
- all merge tasks across ALL unique keys and processes them in a single batch,
- maximizing efficiency for batch operations (e.g., LLM API calls).
-
- Key Features:
- - Tournament algorithm: O(log n) rounds for n duplicate items
- - Batch optimization: Each round processes ALL keys' pairs together
- - Automatic deduplication: Based on key_extractor function
- - Pluggable strategies: Subclasses implement pair_merge() and optionally batch_merge()
-
- Subclass Requirements:
- - Must implement: pair_merge(existing, incoming) -> merged_item
- - Optional override: batch_merge(pairs) -> merged_items (for batch optimization)
-
- Algorithm Example (5 duplicate items for one key):
- Initial: [i1, i2, i3, i4, i5]
-
- Round 1: Pair up [(i1,i2), (i3,i4)], advance [i5]
- → batch_merge([(i1,i2), (i3,i4)]) → [m12, m34]
- → Next round: [m12, m34, i5]
-
- Round 2: Pair up [(m12,m34)], advance [i5]
- → batch_merge([(m12,m34)]) → [m1234]
- → Next round: [m1234, i5]
-
- Round 3: Pair up [(m1234,i5)]
- → batch_merge([(m1234,i5)]) → [final]
-
- Total: 3 rounds, 4 merge operations (all batchable)
-
- Batch Optimization Example:
- - 10 unique keys, each with 4 duplicate items (2 pairs per key)
- - Without batch: 10 keys × 2 pairs = 20 separate API calls
- - With batch: Round 1 collects all 20 pairs → 1 API call
- - Speedup: 20x reduction in API calls
-
- Example:
- >>> class SimpleKeepNewMerger(BaseMerger[Item]):
- ... def pair_merge(self, existing, incoming):
- ... return incoming
- >>>
- >>> merger = SimpleKeepNewMerger(key_extractor=lambda x: x.id)
- >>> merged_items = merger.merge(items)
- """
-
- def __init__(
- self,
- key_extractor: Callable[[T], Any],
- *,
- logger_instance: Optional[logging.Logger] = None,
- ):
- """Initialize the merger.
-
- Args:
- key_extractor: Function to extract unique key from an item.
- Example: lambda x: x.id or lambda x: x.name
- logger_instance: Optional logger instance for progress tracking.
- If None, uses module-level logger.
- """
- self.key_extractor = key_extractor
- self.logger = logger_instance or logger
-
- # ==================== Abstract Methods ====================
-
- @abstractmethod
- def pair_merge(self, existing: T, incoming: T) -> T:
- """Merge two items (must be implemented by subclasses).
-
- This method defines how to merge a pair of duplicate items.
-
- Args:
- existing: The existing item.
- incoming: The incoming item to merge with existing.
-
- Returns:
- The merged item.
-
- Example:
- >>> def pair_merge(self, existing, incoming):
- ... # Keep the newer item
- ... return incoming
- """
- pass
-
- def batch_merge(self, pairs: List[Tuple[T, T]]) -> List[T]:
- """Batch merge multiple pairs (optional override for optimization).
-
- Default implementation: Sequentially call pair_merge() for each pair.
-
- Subclasses should override this method to implement true batch optimization,
- such as batching LLM API calls to reduce latency and cost.
-
- Args:
- pairs: List of (existing, incoming) tuples to merge.
-
- Returns:
- List of merged items (same order as input pairs).
-
- Example (LLM optimization):
- >>> def batch_merge(self, pairs):
- ... # Prepare batch inputs
- ... inputs = [{"a": a.json(), "b": b.json()} for a, b in pairs]
- ... # Single batch API call
- ... return llm_chain.batch(inputs)
- """
- results = []
- for existing, incoming in pairs:
- merged = self.pair_merge(existing, incoming)
- results.append(merged)
- return results
-
- # ==================== Core Algorithm ====================
-
- def merge(self, items: List[T]) -> List[T]:
- """Main entry point: Deduplicate and merge items with cross-key batching.
-
- Algorithm:
- 1. Group items by unique key
- 2. Run tournament merge across ALL keys simultaneously
- 3. Each round collects pairs from ALL keys → single batch_merge() call
- 4. Return list of merged items
-
- This cross-key batching significantly improves efficiency for batch operations
- (e.g., LLM API calls), reducing total API calls by orders of magnitude.
-
- Args:
- items: List of items to merge (may contain duplicates).
-
- Returns:
- List of deduplicated and merged items.
-
- Example:
- >>> items = [
- ... Item(id=1, name="Alice", age=None),
- ... Item(id=1, name=None, age=30),
- ... Item(id=2, name="Bob", age=25),
- ... Item(id=2, name="Bob", city="NYC"),
- ... ]
- >>> merger = FieldMerger(key_extractor=lambda x: x.id)
- >>> merged = merger.merge(items)
- >>> # Round 1: Batch merge [(Alice1, Alice2), (Bob1, Bob2)] in ONE call
- >>> # Result: [Item(id=1, name="Alice", age=30), Item(id=2, name="Bob", city="NYC")]
- """
- if not items:
- return []
-
- # Step 1: Group items by unique key
- groups = self._group_by_key(items)
-
- self.logger.info(f"Grouped {len(items)} items into {len(groups)} unique keys")
-
- # Step 2: Cross-key tournament merge
- merged_items = self._cross_key_tournament_merge(groups)
-
- self.logger.info(f"Merged into {len(merged_items)} unique items")
-
- return merged_items
-
- def _group_by_key(self, items: List[T]) -> Dict[Any, List[T]]:
- """Group items by their unique key.
-
- Args:
- items: List of items to group.
-
- Returns:
- Dictionary mapping key -> list of items with that key.
-
- Example:
- >>> items = [Item(id=1, ...), Item(id=1, ...), Item(id=2, ...)]
- >>> groups = self._group_by_key(items)
- >>> # Result: {1: [Item(id=1, ...), Item(id=1, ...)], 2: [Item(id=2, ...)]}
- """
- groups = defaultdict(list)
- for item in items:
- try:
- key = self.key_extractor(item)
- if key is not None:
- groups[key].append(item)
- else:
- self.logger.warning(f"Item has None key, skipping: {item}")
- except Exception as e:
- self.logger.warning(f"Failed to extract key from item: {e}")
- return dict(groups)
-
- def _cross_key_tournament_merge(self, groups: Dict[Any, List[T]]) -> List[T]:
- """Cross-key tournament merge: Process all keys simultaneously.
-
- **Key Optimization**: Instead of merging each key independently, this method
- runs tournament rounds across ALL keys, collecting all pairs from all keys
- in each round and processing them in a single batch_merge() call.
-
- Performance Impact:
- - Old approach: K keys × log(M) rounds × 1 batch call = K×log(M) API calls
- - New approach: log(max_M) rounds × 1 batch call = log(max_M) API calls
- - Speedup: K× reduction in API calls
-
- Algorithm:
- 1. Initialize each key with its list of items
- 2. Round 1: For each key, pair up items → collect ALL pairs → batch merge
- 3. Round 2: For each key, pair up results → collect ALL pairs → batch merge
- 4. Continue until each key has only 1 item
-
- Args:
- groups: Dictionary mapping key -> list of items with that key.
-
- Returns:
- List of merged items (one per unique key).
-
- Example (2 keys, 4 items each):
- Initial:
- key1: [i1a, i1b, i1c, i1d]
- key2: [i2a, i2b, i2c, i2d]
-
- Round 1: Collect ALL pairs from ALL keys
- key1 pairs: [(i1a,i1b), (i1c,i1d)]
- key2 pairs: [(i2a,i2b), (i2c,i2d)]
- ALL pairs: [(i1a,i1b), (i1c,i1d), (i2a,i2b), (i2c,i2d)]
- batch_merge(ALL 4 pairs) → [m1ab, m1cd, m2ab, m2cd]
- Next round:
- key1: [m1ab, m1cd]
- key2: [m2ab, m2cd]
-
- Round 2: Collect ALL pairs from ALL keys
- key1 pairs: [(m1ab,m1cd)]
- key2 pairs: [(m2ab,m2cd)]
- ALL pairs: [(m1ab,m1cd), (m2ab,m2cd)]
- batch_merge(ALL 2 pairs) → [final1, final2]
-
- Result: [final1, final2]
- Total API calls: 2 (vs 4 in old approach)
- """
- # Track current round of items for each key
- key_rounds: Dict[Any, List[T]] = {
- key: items[:] for key, items in groups.items()
- }
-
- round_num = 0
- max_rounds = max(
- math.ceil(math.log2(len(items))) if len(items) > 1 else 0
- for items in groups.values()
- )
-
- self.logger.debug(
- f"Starting cross-key tournament merge for {len(groups)} keys "
- f"(max {max_rounds} rounds)"
- )
-
- # Continue until all keys have exactly 1 item
- while any(len(items) > 1 for items in key_rounds.values()):
- round_num += 1
- all_pairs = [] # Collect pairs from ALL keys
- pair_to_key = [] # Track which key each pair belongs to
-
- # For each key, pair up items in current round
- for key, current_items in key_rounds.items():
- if len(current_items) <= 1:
- continue # This key is done
-
- # Pair up: (0,1), (2,3), (4,5), ...
- for i in range(0, len(current_items) - 1, 2):
- all_pairs.append((current_items[i], current_items[i + 1]))
- pair_to_key.append(key)
-
- # Batch merge ALL pairs from ALL keys in ONE call
- if all_pairs:
- self.logger.debug(
- f"Round {round_num}/{max_rounds}: Batch merging {len(all_pairs)} pairs "
- f"from {len([k for k, items in key_rounds.items() if len(items) > 1])} keys"
- )
-
- merged_results = self.batch_merge(all_pairs)
-
- # Distribute results back to keys
- result_idx = 0
- for key in key_rounds.keys():
- if len(key_rounds[key]) <= 1:
- continue
-
- next_round = []
-
- # Add merged results for this key's pairs
- key_pair_count = sum(1 for k in pair_to_key if k == key)
- for _ in range(key_pair_count):
- next_round.append(merged_results[result_idx])
- result_idx += 1
-
- # Add odd item if exists
- if len(key_rounds[key]) % 2 == 1:
- next_round.append(key_rounds[key][-1])
-
- key_rounds[key] = next_round
-
- # Extract final merged item for each key
- merged_items = [items[0] for items in key_rounds.values()]
-
- self.logger.debug(f"Cross-key tournament merge completed in {round_num} rounds")
-
- return merged_items
diff --git a/hyperextract/utils/merger/llm.py b/hyperextract/utils/merger/llm.py
deleted file mode 100644
index 2f3932f..0000000
--- a/hyperextract/utils/merger/llm.py
+++ /dev/null
@@ -1,240 +0,0 @@
-"""LLM-powered intelligent merger with batch optimization.
-
-Provides LLMMerger class that uses language models to intelligently merge
-duplicate items. Key features:
-- Batch API calls for efficiency (10-100x speedup)
-- Structured output via Pydantic models
-- Customizable merge prompts
-- Automatic fallback on errors
-"""
-
-import logging
-from typing import List, Tuple, Type, TypeVar, Optional
-from pydantic import BaseModel
-from langchain_core.language_models.chat_models import BaseChatModel
-from langchain_core.prompts import ChatPromptTemplate
-
-from .base import BaseMerger
-
-try:
- from hyperextract.utils.logging import logger
-except ImportError:
- logger = logging.getLogger(__name__)
-
-T = TypeVar("T", bound=BaseModel)
-
-
-class LLMMerger(BaseMerger[T]):
- """LLM-powered intelligent merger with batch optimization.
-
- This merger uses a language model to intelligently merge duplicate items
- by understanding the semantic content and making smart decisions about
- how to combine information from both items.
-
- Key Features:
- - Batch optimization: Overrides batch_merge() to minimize API calls
- - Structured output: Returns Pydantic models matching input schema
- - Custom prompts: Supports user-defined merge instructions
- - Error handling: Automatic fallback to keep_new strategy
-
- Performance:
- - Without batching: N duplicate items = N API calls
- - With batching: N duplicate items = O(log N) API calls
- - Typical speedup: 10-100x reduction in API calls
-
- Requirements:
- - Items must be Pydantic BaseModel instances
- - LLM must support structured output
-
- Example:
- >>> from pydantic import BaseModel
- >>> from langchain_openai import ChatOpenAI
- >>>
- >>> class Person(BaseModel):
- ... name: str
- ... title: str | None = None
- ... skills: list[str] | None = None
- >>>
- >>> llm = ChatOpenAI(model="gpt-4")
- >>> merger = LLMMerger(
- ... key_extractor=lambda x: x.name,
- ... llm_client=llm,
- ... item_schema=Person,
- ... )
- >>>
- >>> items = [
- ... Person(name="孙悟空", title="齐天大圣", skills=None),
- ... Person(name="孙悟空", title=None, skills=["七十二变"]),
- ... Person(name="孙悟空", title=None, skills=["筋斗云"]),
- ... ]
- >>>
- >>> merged = merger.merge(items)
- >>> # LLM intelligently merges: Person(name="孙悟空", title="齐天大圣",
- >>> # skills=["七十二变", "筋斗云"])
- """
-
- def __init__(
- self,
- key_extractor,
- llm_client: BaseChatModel,
- item_schema: Type[T],
- *,
- merge_prompt_template: Optional[str] = None,
- logger_instance: Optional[logging.Logger] = None,
- ):
- """Initialize LLM merger.
-
- Args:
- key_extractor: Function to extract unique key from item.
- llm_client: LangChain chat model instance (must support structured output).
- item_schema: Pydantic model class defining item structure.
- merge_prompt_template: Custom prompt template for merging.
- If None, uses default template.
- logger_instance: Optional logger for progress tracking.
-
- Example:
- >>> merger = LLMMerger(
- ... key_extractor=lambda x: x.id,
- ... llm_client=ChatOpenAI(model="gpt-4"),
- ... item_schema=MyItemSchema,
- ... merge_prompt_template="Merge these items: {item_existing} and {item_incoming}",
- ... )
- """
- super().__init__(key_extractor, logger_instance=logger_instance)
-
- self.llm_client = llm_client
- self.item_schema = item_schema
-
- # Create merge chain with structured output
- template = merge_prompt_template or self._default_merge_template()
- merge_prompt = ChatPromptTemplate.from_template(template)
- self.merge_chain = merge_prompt | llm_client.with_structured_output(item_schema)
-
- @staticmethod
- def _default_merge_template() -> str:
- """Default merge prompt template.
-
- Returns:
- Prompt template string with {item_existing} and {item_incoming} placeholders.
- """
- return """You are an expert at merging structured data intelligently.
-
-Given two instances of the same item (identified by the same unique key),
-your task is to merge their fields into one complete, accurate item.
-
-**Merging Rules:**
-1. Keep the unique key field unchanged (it's the same in both items)
-2. For other fields:
- - If only one item has a non-null value, use that value
- - If both items have values, choose the more complete/accurate one
- - For list fields, combine unique elements from both
- - For text fields, merge or choose the more informative one
-3. Preserve all valuable information from both items
-4. Return a single merged item that represents the best combination
-
-**Item A (existing):**
-{item_existing}
-
-**Item B (incoming):**
-{item_incoming}
-
-**Instructions:**
-Analyze both items carefully and return a single merged item that intelligently
-combines information from both sources. Ensure the output matches the expected schema."""
-
- def pair_merge(self, existing: T, incoming: T) -> T:
- """Merge a single pair using LLM (fallback method).
-
- This method is used as a fallback when batch_merge fails or when
- called directly. For best performance, batch_merge() should be used.
-
- Args:
- existing: The existing item.
- incoming: The incoming item.
-
- Returns:
- Merged item from LLM, or incoming item if LLM fails.
- """
- try:
- self.logger.debug("Performing single LLM merge (fallback)")
- merged = self.merge_chain.invoke(
- {
- "item_existing": existing.model_dump_json(indent=2),
- "item_incoming": incoming.model_dump_json(indent=2),
- }
- )
- return merged
- except Exception as e:
- self.logger.error(
- f"LLM pair merge failed: {e}. Falling back to keep_new strategy."
- )
- return incoming
-
- def batch_merge(self, pairs: List[Tuple[T, T]]) -> List[T]:
- """Batch merge multiple pairs using LLM (optimized).
-
- **This is the key optimization**: Instead of making N separate API calls,
- we batch all pairs into a single API call, reducing latency and cost
- by 10-100x.
-
- Tournament merge algorithm calls this method once per round, with ALL
- pairs from all unique keys in that round. This maximizes batch size
- and minimizes API calls.
-
- Args:
- pairs: List of (existing, incoming) tuples to merge.
-
- Returns:
- List of merged items (same order as input).
-
- Example:
- >>> # Round 1 of tournament merge across 10 keys
- >>> # Each key has 4 items → 2 pairs per key → 20 pairs total
- >>> pairs = [
- ... (item1a, item1b), # Key 1, pair 1
- ... (item1c, item1d), # Key 1, pair 2
- ... (item2a, item2b), # Key 2, pair 1
- ... # ... 17 more pairs from other keys
- ... ]
- >>> # Single batch API call processes all 20 pairs at once
- >>> merged = merger.batch_merge(pairs)
- >>> # Result: 20 merged items in one API call (vs 20 separate calls)
- """
- if not pairs:
- return []
-
- self.logger.info(
- f"Batch merging {len(pairs)} pairs with LLM "
- f"(single API call instead of {len(pairs)} calls)"
- )
-
- # Prepare batch inputs
- inputs = [
- {
- "item_existing": existing.model_dump_json(indent=2),
- "item_incoming": incoming.model_dump_json(indent=2),
- }
- for existing, incoming in pairs
- ]
-
- try:
- # Critical: Single batch API call for all pairs
- merged_results = self.merge_chain.batch(inputs)
-
- self.logger.info(f"Successfully batch merged {len(merged_results)} pairs")
-
- return merged_results
-
- except Exception as e:
- self.logger.error(
- f"Batch LLM merge failed: {e}. "
- f"Falling back to sequential pair_merge for {len(pairs)} pairs."
- )
-
- # Fallback: Sequential pair merges
- results = []
- for existing, incoming in pairs:
- merged = self.pair_merge(existing, incoming)
- results.append(merged)
-
- return results
diff --git a/hyperextract/utils/merger/standard.py b/hyperextract/utils/merger/standard.py
deleted file mode 100644
index feeeb05..0000000
--- a/hyperextract/utils/merger/standard.py
+++ /dev/null
@@ -1,131 +0,0 @@
-"""Standard merge strategies for common use cases.
-
-Provides ready-to-use merger implementations for typical merge scenarios:
-- KeepNewMerger: Always keep the incoming item
-- KeepOldMerger: Always keep the existing item
-- FieldMerger: Merge at field level (Pydantic models)
-"""
-
-import logging
-from typing import List, TypeVar
-from .base import BaseMerger
-
-try:
- from hyperextract.utils.logging import logger
-except ImportError:
- logger = logging.getLogger(__name__)
-
-T = TypeVar("T")
-
-
-class KeepNewMerger(BaseMerger[T]):
- """Merger that always keeps the incoming (newer) item.
-
- Use case: When you want the most recent version of duplicate items.
-
- Example:
- >>> merger = KeepNewMerger(key_extractor=lambda x: x.id)
- >>> items = [Item(id=1, version=1), Item(id=1, version=2)]
- >>> merged = merger.merge(items)
- >>> # Result: [Item(id=1, version=2)]
- """
-
- def pair_merge(self, existing: T, incoming: T) -> T:
- """Keep the incoming item, discard the existing one.
-
- Args:
- existing: The existing item (will be discarded).
- incoming: The incoming item (will be kept).
-
- Returns:
- The incoming item.
- """
- return incoming
-
-
-class KeepOldMerger(BaseMerger[T]):
- """Merger that always keeps the existing (older) item.
-
- Use case: When you want to preserve the first occurrence of duplicate items.
-
- Example:
- >>> merger = KeepOldMerger(key_extractor=lambda x: x.id)
- >>> items = [Item(id=1, version=1), Item(id=1, version=2)]
- >>> merged = merger.merge(items)
- >>> # Result: [Item(id=1, version=1)]
- """
-
- def pair_merge(self, existing: T, incoming: T) -> T:
- """Keep the existing item, discard the incoming one.
-
- Args:
- existing: The existing item (will be kept).
- incoming: The incoming item (will be discarded).
-
- Returns:
- The existing item.
- """
- return existing
-
-
-class FieldMerger(BaseMerger[T]):
- """Merger that combines fields from both items (Pydantic models).
-
- Merge strategy:
- 1. Start with incoming item as base
- 2. Fill in None fields with existing item's non-None values
- 3. Result: Most complete item with fields from both
-
- Requirements:
- - Items must be Pydantic BaseModel instances
- - Items must have compatible schemas
-
- Use case: Accumulating information across multiple extractions.
-
- Example:
- >>> from pydantic import BaseModel
- >>> class Person(BaseModel):
- ... name: str
- ... age: int | None = None
- ... city: str | None = None
- >>>
- >>> merger = FieldMerger(key_extractor=lambda x: x.name)
- >>> items = [
- ... Person(name="Alice", age=30, city=None),
- ... Person(name="Alice", age=None, city="NYC"),
- ... ]
- >>> merged = merger.merge(items)
- >>> # Result: [Person(name="Alice", age=30, city="NYC")]
- """
-
- def pair_merge(self, existing: T, incoming: T) -> T:
- """Merge fields from both items, preferring non-None values.
-
- Strategy:
- - Start with incoming item
- - For each field in existing that is not None:
- - If incoming's field is None, use existing's value
- - Otherwise, keep incoming's value
-
- Args:
- existing: The existing item.
- incoming: The incoming item.
-
- Returns:
- Merged item with fields from both.
-
- Raises:
- AttributeError: If items are not Pydantic models.
- """
- try:
- # Use Pydantic's model_copy with update
- # existing.model_dump(exclude_none=True) gives all non-None fields
- # These values will override incoming's None fields
- return incoming.model_copy(update=existing.model_dump(exclude_none=True))
- except AttributeError as e:
- logger.error(
- f"FieldMerger requires Pydantic BaseModel instances. "
- f"Got types: {type(existing)}, {type(incoming)}. Error: {e}"
- )
- # Fallback to keep incoming
- return incoming