Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
Refactor schema definitions and remove merger utilities
yifanfeng975ecc00f75ca9
diff --git a/hyperextract/templates/en/business/ecosystem.py b/hyperextract/templates/en/business/ecosystem.pyindex ae0fd99..150a350 100644--- a/hyperextract/templates/en/business/ecosystem.py+++ b/hyperextract/templates/en/business/ecosystem.py@@ -8,22 +8,42 @@ from hyperextract.types import AutoHypergraph# 1. Schema Definitions# ==============================================================================+class IndustryParticipant(BaseModel):"""An entity (Company, Regulatory Body, NGO) within an industrial ecosystem."""- name: str = Field(description="Name of the actor (e.g., 'TSMC', 'European Commission').")- rank: Optional[str] = Field(None, description="Market position: 'Tier 1 Supplier', 'Market Leader', 'Challenger'.")- technological_focus: Optional[str] = Field(None, description="Primary tech or domain (e.g., '5G', 'Lithography').")++ name: str = Field(+ description="Name of the actor (e.g., 'TSMC', 'European Commission')."+ )+ rank: Optional[str] = Field(+ None,+ description="Market position: 'Tier 1 Supplier', 'Market Leader', 'Challenger'.",+ )+ technological_focus: Optional[str] = Field(+ None, description="Primary tech or domain (e.g., '5G', 'Lithography')."+ )+class CollectiveAlliance(BaseModel):"""A hyperedge representing a consortium, alliance, market cartel, or joint standard-setting body."""- alliance_id: str = Field(description="Name or identifier of the alliance (e.g., 'CHIPS Alliance').")- participants: List[str] = Field(description="Names of all entities collaborating or competing in this group.")- shared_goal: str = Field(description="The objective: 'Standardization', 'Anti-monopoly Review', 'Joint R&D'.")- governance: Optional[str] = Field(None, description="Rules of participation or leading entity.")++ alliance_id: str = Field(+ description="Name or identifier of the alliance (e.g., 'CHIPS Alliance')."+ )+ participants: List[str] = Field(+ description="Names of all entities collaborating or competing in this group."+ )+ shared_goal: str = Field(+ description="The objective: 'Standardization', 'Anti-monopoly Review', 'Joint R&D'."+ )+ governance: Optional[str] = Field(+ None, description="Rules of participation or leading entity."+ )+# ==============================================================================# 2. Prompts@@ -59,10 +79,12 @@ ECOSYSTEM_EDGE_PROMPT = (# 3. Template Class# ==============================================================================+class MarketEcosystemHyper(AutoHypergraph[IndustryParticipant, CollectiveAlliance]):"""Hypergraph template for industrial clusters, ecosystem coalitions, and complex market alliances."""+def __init__(self,llm_client: BaseChatModel,@@ -73,7 +95,7 @@ class MarketEcosystemHyper(AutoHypergraph[IndustryParticipant, CollectiveAlliancchunk_overlap: int = 256,max_workers: int = 10,verbose: bool = False,- **kwargs: Any+ **kwargs: Any,):super().__init__(node_schema=IndustryParticipant,@@ -91,5 +113,5 @@ class MarketEcosystemHyper(AutoHypergraph[IndustryParticipant, CollectiveAlliancprompt=ECOSYSTEM_CONSOLIDATED_PROMPT,prompt_for_node_extraction=ECOSYSTEM_NODE_PROMPT,prompt_for_edge_extraction=ECOSYSTEM_EDGE_PROMPT,- **kwargs+ **kwargs,)diff --git a/hyperextract/templates/en/business/equity.py b/hyperextract/templates/en/business/equity.pyindex d1b24e8..930c014 100644--- a/hyperextract/templates/en/business/equity.py+++ b/hyperextract/templates/en/business/equity.py@@ -8,24 +8,47 @@ from hyperextract.types import AutoGraph# 1. Schema Definitions# ==============================================================================+class StakeholderNode(BaseModel):"""A legal or natural person holding equity or control in a business."""- name: str = Field(description="Full registry name of the company or individual's full name.")- type: str = Field(description="Type: 'Natural Person', 'Public Company', 'Private Company', 'State-Owned Enterprise', 'Venture Capital'.")- jurisdiction: Optional[str] = Field(None, description="Country or region of incorporation/residence.")- is_ubo: bool = Field(False, description="Whether this entity is identified as an Ultimate Beneficial Owner.")++ name: str = Field(+ description="Full registry name of the company or individual's full name."+ )+ type: str = Field(+ description="Type: 'Natural Person', 'Public Company', 'Private Company', 'State-Owned Enterprise', 'Venture Capital'."+ )+ jurisdiction: Optional[str] = Field(+ None, description="Country or region of incorporation/residence."+ )+ is_ubo: bool = Field(+ False,+ description="Whether this entity is identified as an Ultimate Beneficial Owner.",+ )+class OwnershipRelation(BaseModel):"""A stakeholding or control link between two entities."""+source: str = Field(description="The shareholder or parent entity.")- target: str = Field(description="The company being owned or controlled (subsidiary).")- stake_percentage: Optional[float] = Field(None, description="Percentage of ownership (0.0 to 100.0).")- control_type: str = Field("Equity", description="Nature of link: 'Equity Acquisition', 'Voting Rights', 'Nominee Shareholding', 'Direct Control'.")- as_of_date: Optional[str] = Field(None, description="The date or fiscal period when this ownership was recorded.")+ target: str = Field(+ description="The company being owned or controlled (subsidiary)."+ )+ stake_percentage: Optional[float] = Field(+ None, description="Percentage of ownership (0.0 to 100.0)."+ )+ control_type: str = Field(+ "Equity",+ description="Nature of link: 'Equity Acquisition', 'Voting Rights', 'Nominee Shareholding', 'Direct Control'.",+ )+ as_of_date: Optional[str] = Field(+ None, description="The date or fiscal period when this ownership was recorded."+ )+# ==============================================================================# 2. Prompts@@ -61,10 +84,12 @@ EQUITY_EDGE_PROMPT = (# 3. Template Class# ==============================================================================+class EquityStructureGraph(AutoGraph[StakeholderNode, OwnershipRelation]):"""Precision template for corporate registries, M&A analysis, and KYC/UBO tracking."""+def __init__(self,llm_client: BaseChatModel,@@ -75,13 +100,15 @@ class EquityStructureGraph(AutoGraph[StakeholderNode, OwnershipRelation]):chunk_overlap: int = 256,max_workers: int = 10,verbose: bool = False,- **kwargs: Any+ **kwargs: Any,):super().__init__(node_schema=StakeholderNode,edge_schema=OwnershipRelation,node_key_extractor=lambda x: x.name.strip(),- edge_key_extractor=lambda x: f"{x.source.strip()}->{x.target.strip()}_at_{x.stake_percentage}%",+ edge_key_extractor=lambda x: (+ f"{x.source.strip()}->{x.target.strip()}_at_{x.stake_percentage}%"+ ),nodes_in_edge_extractor=lambda x: (x.source.strip(), x.target.strip()),llm_client=llm_client,embedder=embedder,@@ -93,5 +120,5 @@ class EquityStructureGraph(AutoGraph[StakeholderNode, OwnershipRelation]):prompt=EQUITY_CONSOLIDATED_PROMPT,prompt_for_node_extraction=EQUITY_NODE_PROMPT,prompt_for_edge_extraction=EQUITY_EDGE_PROMPT,- **kwargs+ **kwargs,)diff --git a/hyperextract/templates/en/business/strategy.py b/hyperextract/templates/en/business/strategy.pyindex 3398327..c4988b2 100644--- a/hyperextract/templates/en/business/strategy.py+++ b/hyperextract/templates/en/business/strategy.py@@ -8,24 +8,44 @@ from hyperextract.types import AutoTemporalGraph# 1. Schema Definitions# ==============================================================================+class StrategicNode(BaseModel):"""An organizational entity, business division, or strategic objective."""- name: str = Field(description="Name of the strategic unit or goal (e.g., 'Azure', 'Carbon Neutrality').")- scope: str = Field(description="Scope: 'Global', 'Regional', 'Product-specific', 'Corporate-wide'.")- importance: Optional[str] = Field(None, description="Criticality: 'Core Business', 'Emerging Market', 'Legacy'.")++ name: str = Field(+ description="Name of the strategic unit or goal (e.g., 'Azure', 'Carbon Neutrality')."+ )+ scope: str = Field(+ description="Scope: 'Global', 'Regional', 'Product-specific', 'Corporate-wide'."+ )+ importance: Optional[str] = Field(+ None, description="Criticality: 'Core Business', 'Emerging Market', 'Legacy'."+ )+class StrategicPivot(BaseModel):"""A temporal transition representing a change in strategic focus, investment, or divestiture."""+source: str = Field(description="The original area of focus or acting entity.")target: str = Field(description="The new target, direction, or goal.")- action: str = Field(description="Strategic action: 'Pivot to', 'Acquire', 'Divest', 'Merge', 'Sunset', 'Scale'.")- timestamp: str = Field(description="When the shift occurred (e.g., 'Q4 2022', '2025-01-01').")- rationale: Optional[str] = Field(None, description="The 'Why' behind the move (market pressure, CEO change, tech shift).")- budget: Optional[str] = Field(None, description="Financial commitment mentioned alongside the pivot.")+ action: str = Field(+ description="Strategic action: 'Pivot to', 'Acquire', 'Divest', 'Merge', 'Sunset', 'Scale'."+ )+ timestamp: str = Field(+ description="When the shift occurred (e.g., 'Q4 2022', '2025-01-01')."+ )+ rationale: Optional[str] = Field(+ None,+ description="The 'Why' behind the move (market pressure, CEO change, tech shift).",+ )+ budget: Optional[str] = Field(+ None, description="Financial commitment mentioned alongside the pivot."+ )+# ==============================================================================# 2. Prompts@@ -61,10 +81,12 @@ STRATEGY_EDGE_PROMPT = (# 3. Template Class# ==============================================================================+class StrategicChainGraph(AutoTemporalGraph[StrategicNode, StrategicPivot]):"""Temporal template for tracking corporate strategy, longitudinal organizational change, and market shifts."""+def __init__(self,llm_client: BaseChatModel,@@ -75,13 +97,15 @@ class StrategicChainGraph(AutoTemporalGraph[StrategicNode, StrategicPivot]):chunk_overlap: int = 256,max_workers: int = 10,verbose: bool = False,- **kwargs: Any+ **kwargs: Any,):super().__init__(node_schema=StrategicNode,edge_schema=StrategicPivot,node_key_extractor=lambda x: x.name.strip(),- edge_key_extractor=lambda x: f"{x.source.strip()}-({x.action})->{x.target.strip()}",+ edge_key_extractor=lambda x: (+ f"{x.source.strip()}-({x.action})->{x.target.strip()}"+ ),time_in_edge_extractor=lambda x: x.timestamp.strip(),nodes_in_edge_extractor=lambda x: (x.source.strip(), x.target.strip()),llm_client=llm_client,@@ -94,5 +118,5 @@ class StrategicChainGraph(AutoTemporalGraph[StrategicNode, StrategicPivot]):prompt=STRATEGY_CONSOLIDATED_PROMPT,prompt_for_node_extraction=STRATEGY_NODE_PROMPT,prompt_for_edge_extraction=STRATEGY_EDGE_PROMPT,- **kwargs+ **kwargs,)diff --git a/hyperextract/templates/en/chemistry/molecular.py b/hyperextract/templates/en/chemistry/molecular.pyindex 7117d3b..370e3a1 100644--- a/hyperextract/templates/en/chemistry/molecular.py+++ b/hyperextract/templates/en/chemistry/molecular.py@@ -8,24 +8,41 @@ from hyperextract.types import AutoSpatialGraph# 1. Schema Definitions# ==============================================================================+class AtomNode(BaseModel):"""An individual atom in a chemical structure."""- name: str = Field(description="Atom identifier (e.g., 'C1', 'O2') or element symbol.")++ name: str = Field(+ description="Atom identifier (e.g., 'C1', 'O2') or element symbol."+ )element: str = Field(description="The chemical element (e.g., 'Carbon', 'H').")- hybridization: Optional[str] = Field(None, description="Electronic state: 'sp3', 'sp2', 'sp'.")- formal_charge: int = Field(0, description="The electrical charge assigned to the atom.")+ hybridization: Optional[str] = Field(+ None, description="Electronic state: 'sp3', 'sp2', 'sp'."+ )+ formal_charge: int = Field(+ 0, description="The electrical charge assigned to the atom."+ )+class ChemicalBond(BaseModel):"""A chemical bond between two atoms, including spatial/positional metadata."""+source: str = Field(description="The originating atom (e.g., 'C1').")target: str = Field(description="The connected atom (e.g., 'C2').")- bond_type: str = Field(description="Type: 'Single', 'Double', 'Triple', 'Aromatic', 'Hydrogen'.")- spatial_position: str = Field(description="The physical or logical position in the molecule (e.g., 'C4 position', 'Axial', 'Equatorial').")- bond_length: Optional[float] = Field(None, description="Distance between atoms in Angstroms.")+ bond_type: str = Field(+ description="Type: 'Single', 'Double', 'Triple', 'Aromatic', 'Hydrogen'."+ )+ spatial_position: str = Field(+ description="The physical or logical position in the molecule (e.g., 'C4 position', 'Axial', 'Equatorial')."+ )+ bond_length: Optional[float] = Field(+ None, description="Distance between atoms in Angstroms."+ )+# ==============================================================================# 2. Prompts@@ -61,10 +78,12 @@ MOLECULAR_EDGE_PROMPT = (# 3. Template Class# ==============================================================================+class MolecularStructureGraph(AutoSpatialGraph[AtomNode, ChemicalBond]):"""Spatial graph template for high-resolution molecular modeling and structural biology."""+def __init__(self,llm_client: BaseChatModel,@@ -75,13 +94,15 @@ class MolecularStructureGraph(AutoSpatialGraph[AtomNode, ChemicalBond]):chunk_overlap: int = 256,max_workers: int = 10,verbose: bool = False,- **kwargs: Any+ **kwargs: Any,):super().__init__(node_schema=AtomNode,edge_schema=ChemicalBond,node_key_extractor=lambda x: x.name.strip(),- edge_key_extractor=lambda x: f"{x.source.strip()}--({x.bond_type})-->{x.target.strip()}",+ edge_key_extractor=lambda x: (+ f"{x.source.strip()}--({x.bond_type})-->{x.target.strip()}"+ ),location_in_edge_extractor=lambda x: x.spatial_position.strip(),nodes_in_edge_extractor=lambda x: (x.source.strip(), x.target.strip()),llm_client=llm_client,@@ -94,5 +115,5 @@ class MolecularStructureGraph(AutoSpatialGraph[AtomNode, ChemicalBond]):prompt=MOLECULAR_CONSOLIDATED_PROMPT,prompt_for_node_extraction=MOLECULAR_NODE_PROMPT,prompt_for_edge_extraction=MOLECULAR_EDGE_PROMPT,- **kwargs+ **kwargs,)diff --git a/hyperextract/templates/en/chemistry/protocol.py b/hyperextract/templates/en/chemistry/protocol.pyindex 85185fc..5b657fc 100644--- a/hyperextract/templates/en/chemistry/protocol.py+++ b/hyperextract/templates/en/chemistry/protocol.py@@ -8,25 +8,44 @@ from hyperextract.types import AutoTemporalGraph# 1. Schema Definitions# ==============================================================================+class LabOperation(BaseModel):"""A specific step or action in a chemical or biological protocol (e.g., 'Centrifugation', 'Titration')."""+op_id: str = Field(description="Action name (e.g., 'Centrifuge', 'Heat', 'Add').")- equipment: Optional[str] = Field(None, description="Device used (e.g., 'Microcentrifuge', 'Bunsen burner').")- reagents_added: List[str] = Field(default_factory=list, description="Chemicals introduced during this specific step.")- precautions: Optional[str] = Field(None, description="Safety or accuracy warnings (e.g., 'Keep on ice').")+ equipment: Optional[str] = Field(+ None, description="Device used (e.g., 'Microcentrifuge', 'Bunsen burner')."+ )+ reagents_added: List[str] = Field(+ default_factory=list,+ description="Chemicals introduced during this specific step.",+ )+ precautions: Optional[str] = Field(+ None, description="Safety or accuracy warnings (e.g., 'Keep on ice')."+ )+class ProtocolTransition(BaseModel):"""The sequential transition between protocol steps, capturing duration and order."""+transition_id: str = Field(description="Unique id for the step transition.")source_step: str = Field(description="The preceding operation.")target_step: str = Field(description="The succeeding operation.")- duration: Optional[str] = Field(None, description="Time spent in or between steps (e.g., '15 mins', 'Overnight').")- timestamp: str = Field(description="The sequence order or relative time marker (e.g., 'T+0', 'Step 2').")- wait_condition: Optional[str] = Field(None, description="Condition to proceed (e.g., 'Until precipitate forms').")+ duration: Optional[str] = Field(+ None,+ description="Time spent in or between steps (e.g., '15 mins', 'Overnight').",+ )+ timestamp: str = Field(+ description="The sequence order or relative time marker (e.g., 'T+0', 'Step 2')."+ )+ wait_condition: Optional[str] = Field(+ None, description="Condition to proceed (e.g., 'Until precipitate forms')."+ )+# ==============================================================================# 2. Prompts@@ -63,10 +82,12 @@ PROTOCOL_EDGE_PROMPT = (# 3. Template Class# ==============================================================================+class LabProtocolTemporal(AutoTemporalGraph[LabOperation, ProtocolTransition]):"""Temporal graph template for capturing step-by-step laboratory procedures and experimental workflows."""+def __init__(self,llm_client: BaseChatModel,@@ -77,14 +98,17 @@ class LabProtocolTemporal(AutoTemporalGraph[LabOperation, ProtocolTransition]):chunk_overlap: int = 256,max_workers: int = 10,verbose: bool = False,- **kwargs: Any+ **kwargs: Any,):super().__init__(node_schema=LabOperation,edge_schema=ProtocolTransition,node_key_extractor=lambda x: x.op_id.strip(),edge_key_extractor=lambda x: x.transition_id.strip(),- nodes_in_edge_extractor=lambda x: (x.source_step.strip(), x.target_step.strip()),+ nodes_in_edge_extractor=lambda x: (+ x.source_step.strip(),+ x.target_step.strip(),+ ),time_in_edge_extractor=lambda x: x.timestamp.strip(),llm_client=llm_client,embedder=embedder,@@ -96,5 +120,5 @@ class LabProtocolTemporal(AutoTemporalGraph[LabOperation, ProtocolTransition]):prompt=PROTOCOL_CONSOLIDATED_PROMPT,prompt_for_node_extraction=PROTOCOL_NODE_PROMPT,prompt_for_edge_extraction=PROTOCOL_EDGE_PROMPT,- **kwargs+ **kwargs,)diff --git a/hyperextract/templates/en/chemistry/reaction.py b/hyperextract/templates/en/chemistry/reaction.pyindex 4c2e956..2da5829 100644--- a/hyperextract/templates/en/chemistry/reaction.py+++ b/hyperextract/templates/en/chemistry/reaction.py@@ -8,27 +8,49 @@ from hyperextract.types import AutoHypergraph# 1. Schema Definitions# ==============================================================================+class ChemicalEntity(BaseModel):"""A chemical substance (Reagent, Product, Catalyst, Solvent)."""+name: str = Field(description="IUPAC name or common name (e.g., 'Sulfuric Acid').")- formula: Optional[str] = Field(None, description="Chemical formula (e.g., 'H2SO4').")+ formula: Optional[str] = Field(+ None, description="Chemical formula (e.g., 'H2SO4')."+ )cas_number: Optional[str] = Field(None, description="CAS Registry Number.")- phase: str = Field("Unknown", description="Physical state: 'Gas', 'Liquid', 'Solid', 'Aqueous'.")+ phase: str = Field(+ "Unknown", description="Physical state: 'Gas', 'Liquid', 'Solid', 'Aqueous'."+ )+class ReactionEvent(BaseModel):"""A chemical reaction involving multiple reagents, specific conditions, and resulting products."""- reaction_id: str = Field(description="Name or short description of the reaction (e.g., 'Haber Process').")++ reaction_id: str = Field(+ description="Name or short description of the reaction (e.g., 'Haber Process')."+ )reactants: List[str] = Field(description="List of starting chemical entity names.")products: List[str] = Field(description="List of resulting chemical entity names.")- catalysts: List[str] = Field(default_factory=list, description="Names of substances facilitating the reaction.")- solvents: List[str] = Field(default_factory=list, description="The medium in which the reaction occurs.")- temperature: Optional[str] = Field(None, description="Operating temperature (e.g., '250°C').")- pressure: Optional[str] = Field(None, description="Operating pressure (e.g., '200 atm').")- yield_info: Optional[str] = Field(None, description="Percentage/amount of product obtained.")+ catalysts: List[str] = Field(+ default_factory=list,+ description="Names of substances facilitating the reaction.",+ )+ solvents: List[str] = Field(+ default_factory=list, description="The medium in which the reaction occurs."+ )+ temperature: Optional[str] = Field(+ None, description="Operating temperature (e.g., '250°C')."+ )+ pressure: Optional[str] = Field(+ None, description="Operating pressure (e.g., '200 atm')."+ )+ yield_info: Optional[str] = Field(+ None, description="Percentage/amount of product obtained."+ )+# ==============================================================================# 2. Prompts@@ -64,10 +86,12 @@ REACTION_EDGE_PROMPT = (# 3. Template Class# ==============================================================================+class ChemicalReactionHyper(AutoHypergraph[ChemicalEntity, ReactionEvent]):"""Hypergraph template for complex chemical synthesis, industrial processes, and reaction pathways."""+def __init__(self,llm_client: BaseChatModel,@@ -78,14 +102,16 @@ class ChemicalReactionHyper(AutoHypergraph[ChemicalEntity, ReactionEvent]):chunk_overlap: int = 256,max_workers: int = 10,verbose: bool = False,- **kwargs: Any+ **kwargs: Any,):super().__init__(node_schema=ChemicalEntity,edge_schema=ReactionEvent,node_key_extractor=lambda x: x.name.strip(),edge_key_extractor=lambda x: x.reaction_id.strip(),- nodes_in_edge_extractor=lambda x: tuple(set(x.reactants + x.products + x.catalysts + x.solvents)),+ nodes_in_edge_extractor=lambda x: tuple(+ set(x.reactants + x.products + x.catalysts + x.solvents)+ ),llm_client=llm_client,embedder=embedder,extraction_mode=extraction_mode,@@ -96,5 +122,5 @@ class ChemicalReactionHyper(AutoHypergraph[ChemicalEntity, ReactionEvent]):prompt=REACTION_CONSOLIDATED_PROMPT,prompt_for_node_extraction=REACTION_NODE_PROMPT,prompt_for_edge_extraction=REACTION_EDGE_PROMPT,- **kwargs+ **kwargs,)diff --git a/hyperextract/templates/zh/business/ecosystem.py b/hyperextract/templates/zh/business/ecosystem.pyindex 1a3375c..5b0c32c 100644--- a/hyperextract/templates/zh/business/ecosystem.py+++ b/hyperextract/templates/zh/business/ecosystem.py@@ -8,23 +8,34 @@ from hyperextract.types import AutoHypergraph# 1. Schema Definitions# ==============================================================================+class IndustryParticipant(BaseModel):"""产业生态系统中的实体(公司、监管机构、非政府组织)。"""+name: str = Field(description="参与者名称(如:'台积电'、'欧盟委员会')。")- rank: Optional[str] = Field(None, description="市场地位:'一级供应商'、'市场领导者'、'挑战者'。")- technological_focus: Optional[str] = Field(None, description="主要技术或领域(如:'5G'、'光刻技术')。")+ rank: Optional[str] = Field(+ None, description="市场地位:'一级供应商'、'市场领导者'、'挑战者'。"+ )+ technological_focus: Optional[str] = Field(+ None, description="主要技术或领域(如:'5G'、'光刻技术')。"+ )+class CollectiveAlliance(BaseModel):"""代表财团、联盟、市场卡特尔或联合标准制定机构的超边。"""+alliance_id: str = Field(description="联盟的名称或标识符(如:'芯片联盟')。")- participants: List[str] = Field(description="在该团体中进行合作或竞争的所有实体的名称列表。")+ participants: List[str] = Field(+ description="在该团体中进行合作或竞争的所有实体的名称列表。"+ )shared_goal: str = Field(description="目标:'标准化'、'反垄断审查'、'联合研发'。")governance: Optional[str] = Field(None, description="参与原则或领导实体。")+# ==============================================================================# 2. Prompts# ==============================================================================@@ -59,10 +70,12 @@ ECOSYSTEM_EDGE_PROMPT = (# 3. Template Class# ==============================================================================+class MarketEcosystemHyper(AutoHypergraph[IndustryParticipant, CollectiveAlliance]):"""用于产业集群、生态系统联盟和复杂市场协同的超图模板。"""+def __init__(self,llm_client: BaseChatModel,@@ -73,7 +86,7 @@ class MarketEcosystemHyper(AutoHypergraph[IndustryParticipant, CollectiveAlliancchunk_overlap: int = 256,max_workers: int = 10,verbose: bool = False,- **kwargs: Any+ **kwargs: Any,):super().__init__(node_schema=IndustryParticipant,@@ -91,5 +104,5 @@ class MarketEcosystemHyper(AutoHypergraph[IndustryParticipant, CollectiveAlliancprompt=ECOSYSTEM_CONSOLIDATED_PROMPT,prompt_for_node_extraction=ECOSYSTEM_NODE_PROMPT,prompt_for_edge_extraction=ECOSYSTEM_EDGE_PROMPT,- **kwargs+ **kwargs,)diff --git a/hyperextract/templates/zh/business/equity.py b/hyperextract/templates/zh/business/equity.pyindex ea3973e..8e4c68f 100644--- a/hyperextract/templates/zh/business/equity.py+++ b/hyperextract/templates/zh/business/equity.py@@ -8,24 +8,39 @@ from hyperextract.types import AutoGraph# 1. Schema Definitions# ==============================================================================+class StakeholderNode(BaseModel):"""在业务中持有股份或拥有控制权的法人或自然人。"""+name: str = Field(description="公司注册全称或个人全名。")- type: str = Field(description="类型:'自然人'、'上市公司'、'私有公司'、'国有企业'、'风险投资'。")- jurisdiction: Optional[str] = Field(None, description="注册地或居住地所属的国家或地区。")+ type: str = Field(+ description="类型:'自然人'、'上市公司'、'私有公司'、'国有企业'、'风险投资'。"+ )+ jurisdiction: Optional[str] = Field(+ None, description="注册地或居住地所属的国家或地区。"+ )is_ubo: bool = Field(False, description="该实体是否被识别为最终受益人(UBO)。")+class OwnershipRelation(BaseModel):"""两个实体之间的持股或控制联系。"""+source: str = Field(description="股东或母公司实体。")target: str = Field(description="被拥有或控制的公司(子公司)。")- stake_percentage: Optional[float] = Field(None, description="持股比例(0.0 到 100.0)。")- control_type: str = Field("Equity", description="联系性质:'股权收购'、'投票权'、'代持股份'、'直接控制'。")- as_of_date: Optional[str] = Field(None, description="记录该所有权的时间点或财务周期。")+ stake_percentage: Optional[float] = Field(+ None, description="持股比例(0.0 到 100.0)。"+ )+ control_type: str = Field(+ "Equity", description="联系性质:'股权收购'、'投票权'、'代持股份'、'直接控制'。"+ )+ as_of_date: Optional[str] = Field(+ None, description="记录该所有权的时间点或财务周期。"+ )+# ==============================================================================# 2. Prompts@@ -61,10 +76,12 @@ EQUITY_EDGE_PROMPT = (# 3. Template Class# ==============================================================================+class EquityStructureGraph(AutoGraph[StakeholderNode, OwnershipRelation]):"""用于企业登记、并购分析及 KYC/UBO 追踪的高精度模板。"""+def __init__(self,llm_client: BaseChatModel,@@ -75,13 +92,15 @@ class EquityStructureGraph(AutoGraph[StakeholderNode, OwnershipRelation]):chunk_overlap: int = 256,max_workers: int = 10,verbose: bool = False,- **kwargs: Any+ **kwargs: Any,):super().__init__(node_schema=StakeholderNode,edge_schema=OwnershipRelation,node_key_extractor=lambda x: x.name.strip(),- edge_key_extractor=lambda x: f"{x.source.strip()}->{x.target.strip()}_at_{x.stake_percentage}%",+ edge_key_extractor=lambda x: (+ f"{x.source.strip()}->{x.target.strip()}_at_{x.stake_percentage}%"+ ),nodes_in_edge_extractor=lambda x: (x.source.strip(), x.target.strip()),llm_client=llm_client,embedder=embedder,@@ -93,5 +112,5 @@ class EquityStructureGraph(AutoGraph[StakeholderNode, OwnershipRelation]):prompt=EQUITY_CONSOLIDATED_PROMPT,prompt_for_node_extraction=EQUITY_NODE_PROMPT,prompt_for_edge_extraction=EQUITY_EDGE_PROMPT,- **kwargs+ **kwargs,)diff --git a/hyperextract/templates/zh/business/strategy.py b/hyperextract/templates/zh/business/strategy.pyindex 504a2c7..46d66ea 100644--- a/hyperextract/templates/zh/business/strategy.py+++ b/hyperextract/templates/zh/business/strategy.py@@ -8,25 +8,38 @@ from hyperextract.types import AutoTemporalGraph# 1. Schema Definitions# ==============================================================================+class StrategicNode(BaseModel):"""组织实体、业务部门或战略目标。"""+name: str = Field(description="战略单元或目标的名称(如:'Azure'、'碳中和')。")scope: str = Field(description="范围:'全球'、'区域'、'特定产品'、'集团层面'。")- importance: Optional[str] = Field(None, description="重要性:'核心业务'、'新兴市场'、'传统存量'。")+ importance: Optional[str] = Field(+ None, description="重要性:'核心业务'、'新兴市场'、'传统存量'。"+ )+class StrategicPivot(BaseModel):"""代表战略重点转移、投资或剥离的时间性转变。"""+source: str = Field(description="原始关注领域或施加动作的实体。")target: str = Field(description="新的目标、方向或目的。")- action: str = Field(description="战略动作:'转型为'、'收购'、'剥离'、'合并'、'关停'、'规模化'。")- timestamp: str = Field(description="转型发生的时间(如:'2022年第四季度'、'2025-01-01')。")- rationale: Optional[str] = Field(None, description="举措背后的原因(市场压力、CEO 变动、技术变革)。")+ action: str = Field(+ description="战略动作:'转型为'、'收购'、'剥离'、'合并'、'关停'、'规模化'。"+ )+ timestamp: str = Field(+ description="转型发生的时间(如:'2022年第四季度'、'2025-01-01')。"+ )+ rationale: Optional[str] = Field(+ None, description="举措背后的原因(市场压力、CEO 变动、技术变革)。"+ )budget: Optional[str] = Field(None, description="与转型相关的财务承诺或预算。")+# ==============================================================================# 2. Prompts# ==============================================================================@@ -61,10 +74,12 @@ STRATEGY_EDGE_PROMPT = (# 3. Template Class# ==============================================================================+class StrategicChainGraph(AutoTemporalGraph[StrategicNode, StrategicPivot]):"""用于追踪公司战略、长期组织变革和市场表现的时序模板。"""+def __init__(self,llm_client: BaseChatModel,@@ -75,13 +90,15 @@ class StrategicChainGraph(AutoTemporalGraph[StrategicNode, StrategicPivot]):chunk_overlap: int = 256,max_workers: int = 10,verbose: bool = False,- **kwargs: Any+ **kwargs: Any,):super().__init__(node_schema=StrategicNode,edge_schema=StrategicPivot,node_key_extractor=lambda x: x.name.strip(),- edge_key_extractor=lambda x: f"{x.source.strip()}-({x.action})->{x.target.strip()}",+ edge_key_extractor=lambda x: (+ f"{x.source.strip()}-({x.action})->{x.target.strip()}"+ ),time_in_edge_extractor=lambda x: x.timestamp.strip(),nodes_in_edge_extractor=lambda x: (x.source.strip(), x.target.strip()),llm_client=llm_client,@@ -94,5 +111,5 @@ class StrategicChainGraph(AutoTemporalGraph[StrategicNode, StrategicPivot]):prompt=STRATEGY_CONSOLIDATED_PROMPT,prompt_for_node_extraction=STRATEGY_NODE_PROMPT,prompt_for_edge_extraction=STRATEGY_EDGE_PROMPT,- **kwargs+ **kwargs,)diff --git a/hyperextract/utils/__init__.py b/hyperextract/utils/__init__.pyindex 8e8a47b..5407d0f 100644--- a/hyperextract/utils/__init__.py+++ b/hyperextract/utils/__init__.py@@ -1,22 +1,8 @@"""Hyperextract utilities module."""-from .merger import (- BaseMerger,- MergeStrategy,- KeepNewMerger,- KeepOldMerger,- FieldMerger,- LLMMerger,-)from .logging import logger, setup_logger__all__ = [- "BaseMerger",- "MergeStrategy",- "KeepNewMerger",- "KeepOldMerger",- "FieldMerger",- "LLMMerger","logger","setup_logger",]diff --git a/hyperextract/utils/merger/__init__.py b/hyperextract/utils/merger/__init__.pydeleted file mode 100644index 9f3ba9d..0000000--- a/hyperextract/utils/merger/__init__.py+++ /dev/null@@ -1,18 +0,0 @@-"""Item merger utilities for knowledge patterns.--Provides a flexible merger framework with tournament-style merge algorithm-and support for various merge strategies including LLM-powered intelligent merging.-"""--from .base import BaseMerger, MergeStrategy-from .standard import KeepNewMerger, KeepOldMerger, FieldMerger-from .llm import LLMMerger--__all__ = [- "BaseMerger",- "MergeStrategy",- "KeepNewMerger",- "KeepOldMerger",- "FieldMerger",- "LLMMerger",-]diff --git a/hyperextract/utils/merger/base.py b/hyperextract/utils/merger/base.pydeleted file mode 100644index aee28fd..0000000--- a/hyperextract/utils/merger/base.py+++ /dev/null@@ -1,347 +0,0 @@-"""Base merger class implementing tournament-style merge algorithm.--The tournament merge algorithm efficiently handles n-way merging of duplicate items-by organizing merges into logarithmic rounds, similar to tournament elimination.-"""--import math-import logging-from abc import ABC, abstractmethod-from enum import Enum-from typing import List, Any, Callable, TypeVar, Generic, Dict, Tuple, Optional-from collections import defaultdict--try:- from hyperextract.utils.logging import logger-except ImportError:- logger = logging.getLogger(__name__)--T = TypeVar("T") # Generic item type---class MergeStrategy(str, Enum):- """Merge strategy for handling duplicate items.-- This enum defines the standard strategies for merging duplicate items- when the same unique key is encountered multiple times.-- Strategies:- KEEP_OLD: Keep the existing item, discard the new one- KEEP_NEW: Keep the new item, discard the existing one (default)- FIELD_MERGE: Merge fields from both items (new fills old's None fields)- LLM_MERGE: Use LLM to intelligently merge both items-- Example:- >>> from hyperextract.utils.merger import MergeStrategy- >>> strategy = MergeStrategy.FIELD_MERGE- >>> print(strategy.value) # "field_merge"- """-- KEEP_OLD = "keep_old"- KEEP_NEW = "keep_new"- FIELD_MERGE = "field_merge"- LLM_MERGE = "llm_merge"---class BaseMerger(ABC, Generic[T]):- """Abstract base class for item mergers with tournament-style merge algorithm.-- This class provides the core tournament merge algorithm that efficiently handles- n duplicate items by organizing merges into O(log n) rounds. Each round collects- all merge tasks across ALL unique keys and processes them in a single batch,- maximizing efficiency for batch operations (e.g., LLM API calls).-- Key Features:- - Tournament algorithm: O(log n) rounds for n duplicate items- - Batch optimization: Each round processes ALL keys' pairs together- - Automatic deduplication: Based on key_extractor function- - Pluggable strategies: Subclasses implement pair_merge() and optionally batch_merge()-- Subclass Requirements:- - Must implement: pair_merge(existing, incoming) -> merged_item- - Optional override: batch_merge(pairs) -> merged_items (for batch optimization)-- Algorithm Example (5 duplicate items for one key):- Initial: [i1, i2, i3, i4, i5]-- Round 1: Pair up [(i1,i2), (i3,i4)], advance [i5]- → batch_merge([(i1,i2), (i3,i4)]) → [m12, m34]- → Next round: [m12, m34, i5]-- Round 2: Pair up [(m12,m34)], advance [i5]- → batch_merge([(m12,m34)]) → [m1234]- → Next round: [m1234, i5]-- Round 3: Pair up [(m1234,i5)]- → batch_merge([(m1234,i5)]) → [final]-- Total: 3 rounds, 4 merge operations (all batchable)-- Batch Optimization Example:- - 10 unique keys, each with 4 duplicate items (2 pairs per key)- - Without batch: 10 keys × 2 pairs = 20 separate API calls- - With batch: Round 1 collects all 20 pairs → 1 API call- - Speedup: 20x reduction in API calls-- Example:- >>> class SimpleKeepNewMerger(BaseMerger[Item]):- ... def pair_merge(self, existing, incoming):- ... return incoming- >>>- >>> merger = SimpleKeepNewMerger(key_extractor=lambda x: x.id)- >>> merged_items = merger.merge(items)- """-- def __init__(- self,- key_extractor: Callable[[T], Any],- *,- logger_instance: Optional[logging.Logger] = None,- ):- """Initialize the merger.-- Args:- key_extractor: Function to extract unique key from an item.- Example: lambda x: x.id or lambda x: x.name- logger_instance: Optional logger instance for progress tracking.- If None, uses module-level logger.- """- self.key_extractor = key_extractor- self.logger = logger_instance or logger-- # ==================== Abstract Methods ====================-- @abstractmethod- def pair_merge(self, existing: T, incoming: T) -> T:- """Merge two items (must be implemented by subclasses).-- This method defines how to merge a pair of duplicate items.-- Args:- existing: The existing item.- incoming: The incoming item to merge with existing.-- Returns:- The merged item.-- Example:- >>> def pair_merge(self, existing, incoming):- ... # Keep the newer item- ... return incoming- """- pass-- def batch_merge(self, pairs: List[Tuple[T, T]]) -> List[T]:- """Batch merge multiple pairs (optional override for optimization).-- Default implementation: Sequentially call pair_merge() for each pair.-- Subclasses should override this method to implement true batch optimization,- such as batching LLM API calls to reduce latency and cost.-- Args:- pairs: List of (existing, incoming) tuples to merge.-- Returns:- List of merged items (same order as input pairs).-- Example (LLM optimization):- >>> def batch_merge(self, pairs):- ... # Prepare batch inputs- ... inputs = [{"a": a.json(), "b": b.json()} for a, b in pairs]- ... # Single batch API call- ... return llm_chain.batch(inputs)- """- results = []- for existing, incoming in pairs:- merged = self.pair_merge(existing, incoming)- results.append(merged)- return results-- # ==================== Core Algorithm ====================-- def merge(self, items: List[T]) -> List[T]:- """Main entry point: Deduplicate and merge items with cross-key batching.-- Algorithm:- 1. Group items by unique key- 2. Run tournament merge across ALL keys simultaneously- 3. Each round collects pairs from ALL keys → single batch_merge() call- 4. Return list of merged items-- This cross-key batching significantly improves efficiency for batch operations- (e.g., LLM API calls), reducing total API calls by orders of magnitude.-- Args:- items: List of items to merge (may contain duplicates).-- Returns:- List of deduplicated and merged items.-- Example:- >>> items = [- ... Item(id=1, name="Alice", age=None),- ... Item(id=1, name=None, age=30),- ... Item(id=2, name="Bob", age=25),- ... Item(id=2, name="Bob", city="NYC"),- ... ]- >>> merger = FieldMerger(key_extractor=lambda x: x.id)- >>> merged = merger.merge(items)- >>> # Round 1: Batch merge [(Alice1, Alice2), (Bob1, Bob2)] in ONE call- >>> # Result: [Item(id=1, name="Alice", age=30), Item(id=2, name="Bob", city="NYC")]- """- if not items:- return []-- # Step 1: Group items by unique key- groups = self._group_by_key(items)-- self.logger.info(f"Grouped {len(items)} items into {len(groups)} unique keys")-- # Step 2: Cross-key tournament merge- merged_items = self._cross_key_tournament_merge(groups)-- self.logger.info(f"Merged into {len(merged_items)} unique items")-- return merged_items-- def _group_by_key(self, items: List[T]) -> Dict[Any, List[T]]:- """Group items by their unique key.-- Args:- items: List of items to group.-- Returns:- Dictionary mapping key -> list of items with that key.-- Example:- >>> items = [Item(id=1, ...), Item(id=1, ...), Item(id=2, ...)]- >>> groups = self._group_by_key(items)- >>> # Result: {1: [Item(id=1, ...), Item(id=1, ...)], 2: [Item(id=2, ...)]}- """- groups = defaultdict(list)- for item in items:- try:- key = self.key_extractor(item)- if key is not None:- groups[key].append(item)- else:- self.logger.warning(f"Item has None key, skipping: {item}")- except Exception as e:- self.logger.warning(f"Failed to extract key from item: {e}")- return dict(groups)-- def _cross_key_tournament_merge(self, groups: Dict[Any, List[T]]) -> List[T]:- """Cross-key tournament merge: Process all keys simultaneously.-- **Key Optimization**: Instead of merging each key independently, this method- runs tournament rounds across ALL keys, collecting all pairs from all keys- in each round and processing them in a single batch_merge() call.-- Performance Impact:- - Old approach: K keys × log(M) rounds × 1 batch call = K×log(M) API calls- - New approach: log(max_M) rounds × 1 batch call = log(max_M) API calls- - Speedup: K× reduction in API calls-- Algorithm:- 1. Initialize each key with its list of items- 2. Round 1: For each key, pair up items → collect ALL pairs → batch merge- 3. Round 2: For each key, pair up results → collect ALL pairs → batch merge- 4. Continue until each key has only 1 item-- Args:- groups: Dictionary mapping key -> list of items with that key.-- Returns:- List of merged items (one per unique key).-- Example (2 keys, 4 items each):- Initial:- key1: [i1a, i1b, i1c, i1d]- key2: [i2a, i2b, i2c, i2d]-- Round 1: Collect ALL pairs from ALL keys- key1 pairs: [(i1a,i1b), (i1c,i1d)]- key2 pairs: [(i2a,i2b), (i2c,i2d)]- ALL pairs: [(i1a,i1b), (i1c,i1d), (i2a,i2b), (i2c,i2d)]- batch_merge(ALL 4 pairs) → [m1ab, m1cd, m2ab, m2cd]- Next round:- key1: [m1ab, m1cd]- key2: [m2ab, m2cd]-- Round 2: Collect ALL pairs from ALL keys- key1 pairs: [(m1ab,m1cd)]- key2 pairs: [(m2ab,m2cd)]- ALL pairs: [(m1ab,m1cd), (m2ab,m2cd)]- batch_merge(ALL 2 pairs) → [final1, final2]-- Result: [final1, final2]- Total API calls: 2 (vs 4 in old approach)- """- # Track current round of items for each key- key_rounds: Dict[Any, List[T]] = {- key: items[:] for key, items in groups.items()- }-- round_num = 0- max_rounds = max(- math.ceil(math.log2(len(items))) if len(items) > 1 else 0- for items in groups.values()- )-- self.logger.debug(- f"Starting cross-key tournament merge for {len(groups)} keys "- f"(max {max_rounds} rounds)"- )-- # Continue until all keys have exactly 1 item- while any(len(items) > 1 for items in key_rounds.values()):- round_num += 1- all_pairs = [] # Collect pairs from ALL keys- pair_to_key = [] # Track which key each pair belongs to-- # For each key, pair up items in current round- for key, current_items in key_rounds.items():- if len(current_items) <= 1:- continue # This key is done-- # Pair up: (0,1), (2,3), (4,5), ...- for i in range(0, len(current_items) - 1, 2):- all_pairs.append((current_items[i], current_items[i + 1]))- pair_to_key.append(key)-- # Batch merge ALL pairs from ALL keys in ONE call- if all_pairs:- self.logger.debug(- f"Round {round_num}/{max_rounds}: Batch merging {len(all_pairs)} pairs "- f"from {len([k for k, items in key_rounds.items() if len(items) > 1])} keys"- )-- merged_results = self.batch_merge(all_pairs)-- # Distribute results back to keys- result_idx = 0- for key in key_rounds.keys():- if len(key_rounds[key]) <= 1:- continue-- next_round = []-- # Add merged results for this key's pairs- key_pair_count = sum(1 for k in pair_to_key if k == key)- for _ in range(key_pair_count):- next_round.append(merged_results[result_idx])- result_idx += 1-- # Add odd item if exists- if len(key_rounds[key]) % 2 == 1:- next_round.append(key_rounds[key][-1])-- key_rounds[key] = next_round-- # Extract final merged item for each key- merged_items = [items[0] for items in key_rounds.values()]-- self.logger.debug(f"Cross-key tournament merge completed in {round_num} rounds")-- return merged_itemsdiff --git a/hyperextract/utils/merger/llm.py b/hyperextract/utils/merger/llm.pydeleted file mode 100644index 2f3932f..0000000--- a/hyperextract/utils/merger/llm.py+++ /dev/null@@ -1,240 +0,0 @@-"""LLM-powered intelligent merger with batch optimization.--Provides LLMMerger class that uses language models to intelligently merge-duplicate items. Key features:-- Batch API calls for efficiency (10-100x speedup)-- Structured output via Pydantic models-- Customizable merge prompts-- Automatic fallback on errors-"""--import logging-from typing import List, Tuple, Type, TypeVar, Optional-from pydantic import BaseModel-from langchain_core.language_models.chat_models import BaseChatModel-from langchain_core.prompts import ChatPromptTemplate--from .base import BaseMerger--try:- from hyperextract.utils.logging import logger-except ImportError:- logger = logging.getLogger(__name__)--T = TypeVar("T", bound=BaseModel)---class LLMMerger(BaseMerger[T]):- """LLM-powered intelligent merger with batch optimization.-- This merger uses a language model to intelligently merge duplicate items- by understanding the semantic content and making smart decisions about- how to combine information from both items.-- Key Features:- - Batch optimization: Overrides batch_merge() to minimize API calls- - Structured output: Returns Pydantic models matching input schema- - Custom prompts: Supports user-defined merge instructions- - Error handling: Automatic fallback to keep_new strategy-- Performance:- - Without batching: N duplicate items = N API calls- - With batching: N duplicate items = O(log N) API calls- - Typical speedup: 10-100x reduction in API calls-- Requirements:- - Items must be Pydantic BaseModel instances- - LLM must support structured output-- Example:- >>> from pydantic import BaseModel- >>> from langchain_openai import ChatOpenAI- >>>- >>> class Person(BaseModel):- ... name: str- ... title: str | None = None- ... skills: list[str] | None = None- >>>- >>> llm = ChatOpenAI(model="gpt-4")- >>> merger = LLMMerger(- ... key_extractor=lambda x: x.name,- ... llm_client=llm,- ... item_schema=Person,- ... )- >>>- >>> items = [- ... Person(name="孙悟空", title="齐天大圣", skills=None),- ... Person(name="孙悟空", title=None, skills=["七十二变"]),- ... Person(name="孙悟空", title=None, skills=["筋斗云"]),- ... ]- >>>- >>> merged = merger.merge(items)- >>> # LLM intelligently merges: Person(name="孙悟空", title="齐天大圣",- >>> # skills=["七十二变", "筋斗云"])- """-- def __init__(- self,- key_extractor,- llm_client: BaseChatModel,- item_schema: Type[T],- *,- merge_prompt_template: Optional[str] = None,- logger_instance: Optional[logging.Logger] = None,- ):- """Initialize LLM merger.-- Args:- key_extractor: Function to extract unique key from item.- llm_client: LangChain chat model instance (must support structured output).- item_schema: Pydantic model class defining item structure.- merge_prompt_template: Custom prompt template for merging.- If None, uses default template.- logger_instance: Optional logger for progress tracking.-- Example:- >>> merger = LLMMerger(- ... key_extractor=lambda x: x.id,- ... llm_client=ChatOpenAI(model="gpt-4"),- ... item_schema=MyItemSchema,- ... merge_prompt_template="Merge these items: {item_existing} and {item_incoming}",- ... )- """- super().__init__(key_extractor, logger_instance=logger_instance)-- self.llm_client = llm_client- self.item_schema = item_schema-- # Create merge chain with structured output- template = merge_prompt_template or self._default_merge_template()- merge_prompt = ChatPromptTemplate.from_template(template)- self.merge_chain = merge_prompt | llm_client.with_structured_output(item_schema)-- @staticmethod- def _default_merge_template() -> str:- """Default merge prompt template.-- Returns:- Prompt template string with {item_existing} and {item_incoming} placeholders.- """- return """You are an expert at merging structured data intelligently.--Given two instances of the same item (identified by the same unique key),-your task is to merge their fields into one complete, accurate item.--**Merging Rules:**-1. Keep the unique key field unchanged (it's the same in both items)-2. For other fields:- - If only one item has a non-null value, use that value- - If both items have values, choose the more complete/accurate one- - For list fields, combine unique elements from both- - For text fields, merge or choose the more informative one-3. Preserve all valuable information from both items-4. Return a single merged item that represents the best combination--**Item A (existing):**-{item_existing}--**Item B (incoming):**-{item_incoming}--**Instructions:**-Analyze both items carefully and return a single merged item that intelligently-combines information from both sources. Ensure the output matches the expected schema."""-- def pair_merge(self, existing: T, incoming: T) -> T:- """Merge a single pair using LLM (fallback method).-- This method is used as a fallback when batch_merge fails or when- called directly. For best performance, batch_merge() should be used.-- Args:- existing: The existing item.- incoming: The incoming item.-- Returns:- Merged item from LLM, or incoming item if LLM fails.- """- try:- self.logger.debug("Performing single LLM merge (fallback)")- merged = self.merge_chain.invoke(- {- "item_existing": existing.model_dump_json(indent=2),- "item_incoming": incoming.model_dump_json(indent=2),- }- )- return merged- except Exception as e:- self.logger.error(- f"LLM pair merge failed: {e}. Falling back to keep_new strategy."- )- return incoming-- def batch_merge(self, pairs: List[Tuple[T, T]]) -> List[T]:- """Batch merge multiple pairs using LLM (optimized).-- **This is the key optimization**: Instead of making N separate API calls,- we batch all pairs into a single API call, reducing latency and cost- by 10-100x.-- Tournament merge algorithm calls this method once per round, with ALL- pairs from all unique keys in that round. This maximizes batch size- and minimizes API calls.-- Args:- pairs: List of (existing, incoming) tuples to merge.-- Returns:- List of merged items (same order as input).-- Example:- >>> # Round 1 of tournament merge across 10 keys- >>> # Each key has 4 items → 2 pairs per key → 20 pairs total- >>> pairs = [- ... (item1a, item1b), # Key 1, pair 1- ... (item1c, item1d), # Key 1, pair 2- ... (item2a, item2b), # Key 2, pair 1- ... # ... 17 more pairs from other keys- ... ]- >>> # Single batch API call processes all 20 pairs at once- >>> merged = merger.batch_merge(pairs)- >>> # Result: 20 merged items in one API call (vs 20 separate calls)- """- if not pairs:- return []-- self.logger.info(- f"Batch merging {len(pairs)} pairs with LLM "- f"(single API call instead of {len(pairs)} calls)"- )-- # Prepare batch inputs- inputs = [- {- "item_existing": existing.model_dump_json(indent=2),- "item_incoming": incoming.model_dump_json(indent=2),- }- for existing, incoming in pairs- ]-- try:- # Critical: Single batch API call for all pairs- merged_results = self.merge_chain.batch(inputs)-- self.logger.info(f"Successfully batch merged {len(merged_results)} pairs")-- return merged_results-- except Exception as e:- self.logger.error(- f"Batch LLM merge failed: {e}. "- f"Falling back to sequential pair_merge for {len(pairs)} pairs."- )-- # Fallback: Sequential pair merges- results = []- for existing, incoming in pairs:- merged = self.pair_merge(existing, incoming)- results.append(merged)-- return resultsdiff --git a/hyperextract/utils/merger/standard.py b/hyperextract/utils/merger/standard.pydeleted file mode 100644index feeeb05..0000000--- a/hyperextract/utils/merger/standard.py+++ /dev/null@@ -1,131 +0,0 @@-"""Standard merge strategies for common use cases.--Provides ready-to-use merger implementations for typical merge scenarios:-- KeepNewMerger: Always keep the incoming item-- KeepOldMerger: Always keep the existing item-- FieldMerger: Merge at field level (Pydantic models)-"""--import logging-from typing import List, TypeVar-from .base import BaseMerger--try:- from hyperextract.utils.logging import logger-except ImportError:- logger = logging.getLogger(__name__)--T = TypeVar("T")---class KeepNewMerger(BaseMerger[T]):- """Merger that always keeps the incoming (newer) item.-- Use case: When you want the most recent version of duplicate items.-- Example:- >>> merger = KeepNewMerger(key_extractor=lambda x: x.id)- >>> items = [Item(id=1, version=1), Item(id=1, version=2)]- >>> merged = merger.merge(items)- >>> # Result: [Item(id=1, version=2)]- """-- def pair_merge(self, existing: T, incoming: T) -> T:- """Keep the incoming item, discard the existing one.-- Args:- existing: The existing item (will be discarded).- incoming: The incoming item (will be kept).-- Returns:- The incoming item.- """- return incoming---class KeepOldMerger(BaseMerger[T]):- """Merger that always keeps the existing (older) item.-- Use case: When you want to preserve the first occurrence of duplicate items.-- Example:- >>> merger = KeepOldMerger(key_extractor=lambda x: x.id)- >>> items = [Item(id=1, version=1), Item(id=1, version=2)]- >>> merged = merger.merge(items)- >>> # Result: [Item(id=1, version=1)]- """-- def pair_merge(self, existing: T, incoming: T) -> T:- """Keep the existing item, discard the incoming one.-- Args:- existing: The existing item (will be kept).- incoming: The incoming item (will be discarded).-- Returns:- The existing item.- """- return existing---class FieldMerger(BaseMerger[T]):- """Merger that combines fields from both items (Pydantic models).-- Merge strategy:- 1. Start with incoming item as base- 2. Fill in None fields with existing item's non-None values- 3. Result: Most complete item with fields from both-- Requirements:- - Items must be Pydantic BaseModel instances- - Items must have compatible schemas-- Use case: Accumulating information across multiple extractions.-- Example:- >>> from pydantic import BaseModel- >>> class Person(BaseModel):- ... name: str- ... age: int | None = None- ... city: str | None = None- >>>- >>> merger = FieldMerger(key_extractor=lambda x: x.name)- >>> items = [- ... Person(name="Alice", age=30, city=None),- ... Person(name="Alice", age=None, city="NYC"),- ... ]- >>> merged = merger.merge(items)- >>> # Result: [Person(name="Alice", age=30, city="NYC")]- """-- def pair_merge(self, existing: T, incoming: T) -> T:- """Merge fields from both items, preferring non-None values.-- Strategy:- - Start with incoming item- - For each field in existing that is not None:- - If incoming's field is None, use existing's value- - Otherwise, keep incoming's value-- Args:- existing: The existing item.- incoming: The incoming item.-- Returns:- Merged item with fields from both.-- Raises:- AttributeError: If items are not Pydantic models.- """- try:- # Use Pydantic's model_copy with update- # existing.model_dump(exclude_none=True) gives all non-None fields- # These values will override incoming's None fields- return incoming.model_copy(update=existing.model_dump(exclude_none=True))- except AttributeError as e:- logger.error(- f"FieldMerger requires Pydantic BaseModel instances. "- f"Got types: {type(existing)}, {type(incoming)}. Error: {e}"- )- # Fallback to keep incoming- return incoming