Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
Add merger utilities with tournament-style algorithm and LLM integration
yifanfeng97d310ec6b030d
diff --git a/examples/set_knowledge_demo.py b/examples/set_knowledge_demo.pyindex cf3255f..b372453 100644--- a/examples/set_knowledge_demo.py+++ b/examples/set_knowledge_demo.py@@ -15,7 +15,8 @@ import osfrom pydantic import BaseModel, Fieldfrom langchain_openai import ChatOpenAI, OpenAIEmbeddings-from hyperextract.knowledge.generic.set import SetKnowledge, MergeItemStrategy+from hyperextract.knowledge.generic.set import SetKnowledge+from hyperextract.utils.merger import MergeStrategy# Define the schema for skills@@ -50,8 +51,8 @@ def main():item_schema=SkillSchema,llm_client=llm,embedder=embedder,- unique_key="name", # Deduplicate based on skill name- merge_item_strategy=MergeItemStrategy.FIELD_MERGE,+ key_extractor=lambda x: x.name, # Deduplicate based on skill name+ merge_item_strategy=MergeStrategy.FIELD_MERGE,prompt="""Extract all technical skills, frameworks, tools, and soft skills mentioned in the text.For each skill, provide:@@ -194,7 +195,7 @@ Be specific and extract all mentioned skills.""",item_schema=SkillSchema,llm_client=llm,embedder=embedder,- unique_key="name",+ key_extractor=lambda x: x.name,)loaded_skills.load(save_path)print(f"✓ Loaded {len(loaded_skills)} unique skills")@@ -206,7 +207,7 @@ Be specific and extract all mentioned skills.""",print(f" Total unique skills extracted: {len(skills)}")print(f" Job descriptions processed: {len(job_descriptions)}")print(" Deduplication strategy: FIELD_MERGE")- print(" Unique key: 'name'")+ print(" Key extractor: lambda x: x.name")print("\n ✓ Automatically deduplicated based on skill names")print(" ✓ Merged information from multiple occurrences")print(" ✓ Supports semantic search across all skills")diff --git a/hyperextract/knowledge/generic/__init__.py b/hyperextract/knowledge/generic/__init__.pyindex 2e4df7a..79fca3e 100644--- a/hyperextract/knowledge/generic/__init__.py+++ b/hyperextract/knowledge/generic/__init__.py@@ -2,12 +2,11 @@from .unit import UnitKnowledgefrom .list import ListKnowledge, ItemListSchema-from .set import SetKnowledge, MergeItemStrategy+from .set import SetKnowledge__all__ = ["UnitKnowledge","ListKnowledge","ItemListSchema","SetKnowledge",- "MergeItemStrategy",]diff --git a/hyperextract/knowledge/generic/set.py b/hyperextract/knowledge/generic/set.pyindex b381b6f..e551691 100644--- a/hyperextract/knowledge/generic/set.py+++ b/hyperextract/knowledge/generic/set.py@@ -5,17 +5,23 @@ Supports multiple merge strategies including LLM-powered intelligent merging."""import json-from typing import List, Any, Type, TypeVar, Generic, Dict, Optional+from typing import List, Any, Type, TypeVar, Generic, Dict, Optional, Callablefrom datetime import datetimefrom pathlib import Path-from enum import Enumfrom pydantic import BaseModel, Field, create_modelfrom langchain_core.language_models.chat_models import BaseChatModelfrom langchain_core.embeddings import Embeddings-from langchain_core.prompts import ChatPromptTemplatefrom langchain_core.documents import Documentfrom ..base import BaseKnowledge+from hyperextract.utils.merger import (+ BaseMerger,+ MergeStrategy,+ KeepNewMerger,+ KeepOldMerger,+ FieldMerger,+ LLMMerger,+)try:from hyperextract.config import logger@@ -34,32 +40,16 @@ class ItemSetSchema(BaseModel, Generic[Item]):items: List[Item] = Field(default_factory=list, description="Set of unique items")-class MergeItemStrategy(str, Enum):- """Merge strategy for handling duplicate items.-- Strategies:- KEEP_OLD: Keep the existing item, discard the new one- KEEP_NEW: Keep the new item, discard the existing one (default)- FIELD_MERGE: Merge fields from both items (new fills old's None fields)- LLM_MERGE: Use LLM to intelligently merge both items- """-- KEEP_OLD = "keep_old"- KEEP_NEW = "keep_new"- FIELD_MERGE = "field_merge"- LLM_MERGE = "llm_merge"--class SetKnowledge(BaseKnowledge[ItemSetSchema[Item]], Generic[Item]):"""Set Knowledge Pattern - extracts a unique collection of objects.This pattern automatically deduplicates items based on a user-specified- unique key field. Provides flexible merge strategies including LLM-powered+ key extractor function. Provides flexible merge strategies including LLM-poweredintelligent merging for handling duplicates.Key characteristics:- Extraction target: A unique collection of structured objects- - Deduplication: Based on unique_key field (user-specified)+ - Deduplication: Based on key_extractor function (user-specified)- Merge strategy: Configurable (keep_old/keep_new/field_merge/llm_merge)- Internal storage: Dict for O(1) lookup and deduplication- External interface: List (via items property)@@ -79,7 +69,7 @@ class SetKnowledge(BaseKnowledge[ItemSetSchema[Item]], Generic[Item]):... item_schema=KeywordSchema,... llm_client=llm,... embedder=embedder,- ... unique_key="term",+ ... key_extractor=lambda x: x.term,... merge_item_strategy="field_merge"... )>>> keywords.extract("Python is great. Python is powerful.")@@ -93,8 +83,8 @@ class SetKnowledge(BaseKnowledge[ItemSetSchema[Item]], Generic[Item]):llm_client: BaseChatModel,embedder: Embeddings,*,- unique_key: str,- merge_item_strategy: MergeItemStrategy = MergeItemStrategy.KEEP_NEW,+ key_extractor: Callable[[Item], Any],+ merge_item_strategy: MergeStrategy = MergeStrategy.KEEP_NEW,prompt: str = "",chunk_size: int = 2000,chunk_overlap: int = 200,@@ -103,13 +93,13 @@ class SetKnowledge(BaseKnowledge[ItemSetSchema[Item]], Generic[Item]):llm_batch_size: int = 10,**kwargs,):- """Initialize SetKnowledge with unique key and merge strategy.+ """Initialize SetKnowledge with key extractor and merge strategy.Args:item_schema: Pydantic BaseModel subclass for individual items.llm_client: Language model client for extraction and merging.embedder: Embedding model for vector indexing.- unique_key: Field name to use as unique identifier (required).+ key_extractor: Function to extract unique key from an item (required).merge_item_strategy: Strategy for merging duplicate items (default: KEEP_NEW).prompt: Custom extraction prompt.chunk_size: Maximum characters per chunk for long texts.@@ -118,13 +108,7 @@ class SetKnowledge(BaseKnowledge[ItemSetSchema[Item]], Generic[Item]):show_progress: Whether to log progress information.llm_batch_size: Batch size for LLM merge operations (for LLM_MERGE strategy).**kwargs: Additional arguments passed to parent class.-- Raises:- ValueError: If unique_key field does not exist in item_schema.- TypeError: If unique_key field type is not hashable."""- # Validate unique_key before initialization- self._validate_unique_key(item_schema, unique_key)# Store item_schemaself.item_schema = item_schema@@ -153,18 +137,18 @@ class SetKnowledge(BaseKnowledge[ItemSetSchema[Item]], Generic[Item]):)# SetKnowledge-specific attributes- self.unique_key = unique_key+ self.key_extractor = key_extractorself.merge_item_strategy = merge_item_strategyself.llm_batch_size = llm_batch_size# Internal storage: Dict for O(1) deduplicationself._items_dict: Dict[Any, Item] = {}- # Create LLM merge chain (if using LLM_MERGE strategy)- self._merge_chain = (- self._create_merge_chain()- if merge_item_strategy == MergeItemStrategy.LLM_MERGE- else None+ # Create Merger instance based on strategy+ self._merger: BaseMerger[Item] = self._create_merger(+ item_schema=item_schema,+ llm_client=llm_client,+ merge_item_strategy=merge_item_strategy,)# ==================== Override Instance Creation ====================@@ -181,7 +165,7 @@ class SetKnowledge(BaseKnowledge[ItemSetSchema[Item]], Generic[Item]):item_schema=self.item_schema,llm_client=self.llm_client,embedder=self.embedder,- unique_key=self.unique_key,+ key_extractor=self.key_extractor,merge_item_strategy=self.merge_item_strategy,prompt=self.prompt,chunk_size=self.chunk_size,@@ -204,78 +188,48 @@ class SetKnowledge(BaseKnowledge[ItemSetSchema[Item]], Generic[Item]):"### Source Text:\n")- @staticmethod- def _validate_unique_key(item_schema: Type[Item], unique_key: str):- """Validates that unique_key exists in schema and is hashable.-- Args:- item_schema: The Pydantic schema to validate.- unique_key: The field name to validate.-- Raises:- ValueError: If unique_key field does not exist.- TypeError: If unique_key field type is known to be unhashable.- """- # Check field existence- if unique_key not in item_schema.model_fields:- available_fields = list(item_schema.model_fields.keys())- raise ValueError(- f"unique_key '{unique_key}' not found in {item_schema.__name__}. "- f"Available fields: {available_fields}"- )-- # Check field type (basic unhashable type detection)- field_info = item_schema.model_fields[unique_key]- field_type = field_info.annotation-- # Extract actual type from Optional/Union if present- import typing-- if (- hasattr(typing, "get_origin")- and typing.get_origin(field_type) is typing.Union- ):- args = typing.get_args(field_type)- field_type = args[0] if args else field_type-- # Check for known unhashable types- unhashable_types = (list, dict, set, List, Dict)- if field_type in unhashable_types:- raise TypeError(- f"unique_key field '{unique_key}' has unhashable type {field_type}. "- f"unique_key must be a hashable type (str, int, tuple, etc.)"- )-- # ==================== LLM Merge Chain ====================+ # ==================== Merger Factory ====================- def _create_merge_chain(self):- """Creates an LLM chain for intelligently merging duplicate items.+ def _create_merger(+ self,+ item_schema: Type[Item],+ llm_client: BaseChatModel,+ merge_item_strategy: MergeStrategy,+ ) -> BaseMerger[Item]:+ """Creates the appropriate Merger instance based on strategy.- Subclasses can override this method to customize the merge prompt.+ Args:+ item_schema: Pydantic model for items.+ llm_client: LLM client for LLM_MERGE strategy.+ merge_item_strategy: The merge strategy to use.Returns:- LangChain chain for merging items.+ A Merger instance implementing the specified strategy."""- merge_template = """You are an expert at merging structured data.-Given two instances of the same item (identified by the same unique key),-intelligently merge their fields into one complete item.+ key_extractor = self.key_extractor-Merging rules:-1. Keep the unique key field unchanged-2. For other fields: prefer non-null values-3. If both items have values for a field, choose the more complete/accurate one-4. Combine text fields if appropriate (e.g., descriptions)+ if merge_item_strategy == MergeStrategy.KEEP_OLD:+ return KeepOldMerger(key_extractor, logger_instance=logger)-Item A (existing):-{item_existing}+ elif merge_item_strategy == MergeStrategy.KEEP_NEW:+ return KeepNewMerger(key_extractor, logger_instance=logger)-Item B (incoming):-{item_incoming}+ elif merge_item_strategy == MergeStrategy.FIELD_MERGE:+ return FieldMerger(key_extractor, logger_instance=logger)-Please merge these two items intelligently and return the merged result."""+ elif merge_item_strategy == MergeStrategy.LLM_MERGE:+ return LLMMerger(+ key_extractor=key_extractor,+ llm_client=llm_client,+ item_schema=item_schema,+ logger_instance=logger,+ )- merge_prompt = ChatPromptTemplate.from_template(merge_template)- return merge_prompt | self.llm_client.with_structured_output(self.item_schema)+ else:+ logger.warning(+ f"Unknown merge strategy: {merge_item_strategy}, using KEEP_NEW"+ )+ return KeepNewMerger(key_extractor, logger_instance=logger)# ==================== Properties ====================@@ -387,12 +341,10 @@ Please merge these two items intelligently and return the merged result."""# If store=True, merge with existing data and update internal stateif store:if self._data and len(self.items) > 0:- self._data = self.merge([self._data, merged_data])+ final_data = self.merge([self._data, merged_data])else:- self._data = merged_data-- self.clear_index()- self.metadata["updated_at"] = datetime.now()+ final_data = merged_data+ self._update_internal_state(final_data)logger.info("Knowledge extraction completed")logger.info(@@ -405,97 +357,61 @@ Please merge these two items intelligently and return the merged result."""def merge(self, data_list: List[ItemSetSchema[Item]]) -> ItemSetSchema[Item]:"""Merges multiple data containers with automatic deduplication.- Overrides parent's merge to implement set semantics:- 1. Collect all items from all containers- 2. For each item, check if unique_key already exists- 3. If duplicate, call merge_item() to resolve- 4. Return deduplicated result-- For LLM_MERGE strategy, batches duplicate pairs for efficient processing.+ Pure function: Does not modify internal state.+ Delegates to Merger for efficient tournament-style merging.+ All merge strategies (including LLM batch processing) are handled+ by the Merger implementation.Args:data_list: List of container objects to merge.Returns:- Merged ItemSetSchema with deduplicated items.+ New merged ItemSetSchema with deduplicated items."""- # Collect all items and track duplicates for batch LLM merge- merge_pairs = [] # [(existing, incoming, key), ...]-+ # Step 1: Collect all items from all containers+ all_items = []for data in data_list:- for item in data.items:- # Get unique key value- key_value = self._get_key_value(item)- if key_value is None:- continue-- if key_value in self._items_dict:- # Duplicate found- if self.merge_item_strategy == MergeItemStrategy.LLM_MERGE:- # Collect for batch processing- merge_pairs.append(- (self._items_dict[key_value], item, key_value)- )- else:- # Merge immediately with non-LLM strategies- merged = self.merge_item(self._items_dict[key_value], item)- if merged is not None:- self._items_dict[key_value] = merged- else:- # New item- self._items_dict[key_value] = item.model_copy(deep=True)-- # Batch process LLM merges if any- if merge_pairs and self.merge_item_strategy == MergeItemStrategy.LLM_MERGE:- merged_results = self._batch_merge_with_llm(merge_pairs)- # Update items_dict with merged results- self._items_dict.update(merged_results)-- # Update internal _data to match _items_dict- self._data.items = list(self._items_dict.values())+ all_items.extend(data.items)++ if not all_items:+ return self.item_set_schema()logger.info(- f"Merged into {len(self._items_dict)} unique items "+ f"Merging {len(all_items)} items from {len(data_list)} containers..."+ )++ # Step 2: Use Merger to deduplicate and merge items+ merged_items = self._merger.merge(all_items)++ logger.info(+ f"Merged into {len(merged_items)} unique items "f"(strategy: {self.merge_item_strategy.value})")- return self._data+ # Step 3: Return new container (does not modify self)+ return self.item_set_schema(items=merged_items)- def merge_item(self, existing: Item, incoming: Item) -> Optional[Item]:- """Merges two duplicate items based on the configured strategy.+ def _update_internal_state(self, data: ItemSetSchema[Item]) -> None:+ """Updates internal state from a data container.- This method can be overridden by subclasses to implement custom merge logic.+ Private method that synchronizes _items_dict and _data with new data.+ Also clears vector index and updates metadata timestamp.Args:- existing: The existing item in the set.- incoming: The incoming item to merge.-- Returns:- Merged item, or None to skip this item.+ data: The data container to update from."""- if self.merge_item_strategy == MergeItemStrategy.KEEP_OLD:- return existing-- elif self.merge_item_strategy == MergeItemStrategy.KEEP_NEW:- return incoming-- elif self.merge_item_strategy == MergeItemStrategy.FIELD_MERGE:- # Field-level merge: existing's non-None values override incoming- return incoming.model_copy(update=existing.model_dump(exclude_none=True))-- elif self.merge_item_strategy == MergeItemStrategy.LLM_MERGE:- # LLM merge (should be handled by batch processing in merge())- # Fallback to single merge if called directly- return self._merge_item_with_llm(existing, incoming)+ self._items_dict.clear()+ for item in data.items:+ key = self._get_key_value(item)+ if key is not None:+ self._items_dict[key] = item- else:- logger.warning(- f"Unknown merge strategy: {self.merge_item_strategy}, using KEEP_NEW"- )- return incoming+ self._data = data+ self.clear_index()+ self.metadata["updated_at"] = datetime.now()def _get_key_value(self, item: Item) -> Optional[Any]:- """Extracts the unique key value from an item.+ """Extracts the unique key value from an item using key_extractor.Args:item: The item to extract key from.@@ -504,114 +420,25 @@ Please merge these two items intelligently and return the merged result."""The unique key value, or None if invalid."""try:- key_value = getattr(item, self.unique_key)+ key_value = self.key_extractor(item)# Validate key valueif key_value is None:- logger.warning(- f"Item has None value for unique_key '{self.unique_key}', skipping"- )+ logger.warning("Item has None value for key, skipping")return None# Verify hashabilityhash(key_value)return key_value- except TypeError:- logger.error(- f"Value {key_value} for unique_key '{self.unique_key}' is not hashable"- )+ except TypeError as e:+ logger.error(f"Key value is not hashable: {e}")return None- except AttributeError:- logger.error(f"Item missing unique_key field '{self.unique_key}'")+ except Exception as e:+ logger.error(f"Failed to extract key from item: {e}")return None- # ==================== LLM Merge Methods ====================-- def _batch_merge_with_llm(self, merge_pairs: List[tuple]) -> Dict[Any, Item]:- """Batch processes LLM merges for efficiency.-- Args:- merge_pairs: List of (existing, incoming, key_value) tuples to merge.-- Returns:- Dictionary mapping key_value to merged items.- """- results = {}-- if not self._merge_chain:- logger.warning("LLM merge chain not initialized, falling back to KEEP_NEW")- for existing, incoming, key_value in merge_pairs:- results[key_value] = incoming- return results-- total_pairs = len(merge_pairs)- logger.info(f"Batch merging {total_pairs} duplicate items with LLM...")-- # Process in batches- for i in range(0, total_pairs, self.llm_batch_size):- batch = merge_pairs[i : i + self.llm_batch_size]-- # Prepare batch inputs- inputs = [- {- "item_existing": existing.model_dump_json(indent=2),- "item_incoming": incoming.model_dump_json(indent=2),- }- for existing, incoming, _ in batch- ]-- try:- # Batch invoke LLM- merged_results = self._merge_chain.batch(inputs)-- # Collect merged results- for (existing, incoming, key_value), merged in zip(- batch, merged_results- ):- if merged:- results[key_value] = merged- logger.debug(f"Successfully merged item with key '{key_value}'")- else:- logger.warning(- f"LLM merge returned None for key '{key_value}', keeping incoming"- )- results[key_value] = incoming-- except Exception as e:- logger.error(f"Batch LLM merge failed: {e}, falling back to KEEP_NEW")- # Fallback: keep incoming items- for existing, incoming, key_value in batch:- results[key_value] = incoming-- return results-- def _merge_item_with_llm(self, existing: Item, incoming: Item) -> Optional[Item]:- """Uses LLM to intelligently merge two duplicate items (single merge).-- Args:- existing: The existing item.- incoming: The incoming item to merge.-- Returns:- Merged item, or None on failure.- """- if not self._merge_chain:- logger.warning("LLM merge chain not initialized")- return incoming-- try:- merged = self._merge_chain.invoke(- {- "item_existing": existing.model_dump_json(indent=2),- "item_incoming": incoming.model_dump_json(indent=2),- }- )- logger.debug("Successfully merged item with LLM")- return merged- except Exception as e:- logger.error(f"LLM merge failed: {e}")- return incoming # Fallback to incoming item+ # ==================== LLM Merge Methods (Removed - Now handled by LLMMerger) ====================def build_index(self):"""Builds independent vector index for each unique item."""@@ -694,7 +521,6 @@ Please merge these two items intelligently and return the merged result.""""schema_name": self.item_set_schema.__name__,"item_schema_name": self.item_schema.__name__,"item_schema": self.item_schema.model_json_schema(),- "unique_key": self.unique_key,"merge_item_strategy": self.merge_item_strategy.value,"data": self.data.model_dump(),"metadata": {@@ -736,16 +562,10 @@ Please merge these two items intelligently and return the merged result."""data = json.load(f)logger.info(f"Loaded data from {data_file}")- self._data = self.item_set_schema.model_validate(data.get("data", {}))+ loaded_data = self.item_set_schema.model_validate(data.get("data", {}))+ self._update_internal_state(loaded_data)- # Rebuild items_dict from _data.items (not self.items property!)- self._items_dict = {}- for item in self._data.items:- key_value = self._get_key_value(item)- if key_value is not None:- self._items_dict[key_value] = item-- # Update metadata+ # Update metadata from saved dataif "metadata" in data:self.metadata.update(data["metadata"])self.metadata["updated_at"] = datetime.now()@@ -783,16 +603,17 @@ Please merge these two items intelligently and return the merged result."""if key_value in self._items_dict:# Merge with existing- merged = self.merge_item(self._items_dict[key_value], item)+ merged = self._merger.pair_merge(self._items_dict[key_value], item)if merged:self._items_dict[key_value] = mergedelse:# Add newself._items_dict[key_value] = item.model_copy(deep=True)- # Update _data+ # Update internal stateself._data.items = list(self._items_dict.values())self.clear_index()+ self.metadata["updated_at"] = datetime.now()def remove(self, key: Any) -> Optional[Item]:"""Removes an item by its unique key value.@@ -807,6 +628,7 @@ Please merge these two items intelligently and return the merged result."""if removed:self._data.items = list(self._items_dict.values())self.clear_index()+ self.metadata["updated_at"] = datetime.now()logger.debug(f"Removed item with key '{key}'")return removed@@ -857,8 +679,8 @@ Please merge these two items intelligently and return the merged result."""- Clears the vector index (needs rebuild)- Updates metadata timestamp"""- removed = self._items_dict.pop(key, None)- if removed:+ if key in self._items_dict:+ self._items_dict.pop(key)self._data.items = list(self._items_dict.values())self.clear_index()self.metadata["updated_at"] = datetime.now()@@ -946,7 +768,7 @@ Please merge these two items intelligently and return the merged result."""# Add items from other (merge duplicates)for key, item in other._items_dict.items():if key in new_set._items_dict:- merged = new_set.merge_item(new_set._items_dict[key], item)+ merged = new_set._merger.pair_merge(new_set._items_dict[key], item)if merged:new_set._items_dict[key] = mergedelse:@@ -986,7 +808,7 @@ Please merge these two items intelligently and return the merged result."""for key in self._items_dict:if key in other._items_dict:# Merge the two versions- merged = new_set.merge_item(+ merged = new_set._merger.pair_merge(self._items_dict[key], other._items_dict[key])if merged:@@ -1095,7 +917,7 @@ Please merge these two items intelligently and return the merged result."""def __eq__(self, other: Any) -> bool:"""Equality comparison: set1 == set2.- Two sets are equal if they have the same schema, unique_key, and key set.+ Two sets are equal if they have the same schema and key set.Note: Does not compare item contents, only keys.Args:@@ -1113,9 +935,6 @@ Please merge these two items intelligently and return the merged result."""if self.item_schema != other.item_schema:return False- if self.unique_key != other.unique_key:- return False-return set(self._items_dict.keys()) == set(other._items_dict.keys())def __ne__(self, other: Any) -> bool:diff --git a/hyperextract/utils/__init__.py b/hyperextract/utils/__init__.pynew file mode 100644index 0000000..596ffa7--- /dev/null+++ b/hyperextract/utils/__init__.py@@ -0,0 +1,19 @@+"""Hyperextract utilities module."""++from .merger import (+ BaseMerger,+ MergeStrategy,+ KeepNewMerger,+ KeepOldMerger,+ FieldMerger,+ LLMMerger,+)++__all__ = [+ "BaseMerger",+ "MergeStrategy",+ "KeepNewMerger",+ "KeepOldMerger",+ "FieldMerger",+ "LLMMerger",+]diff --git a/hyperextract/utils/merger/__init__.py b/hyperextract/utils/merger/__init__.pynew file mode 100644index 0000000..9f3ba9d--- /dev/null+++ b/hyperextract/utils/merger/__init__.py@@ -0,0 +1,18 @@+"""Item merger utilities for knowledge patterns.++Provides a flexible merger framework with tournament-style merge algorithm+and support for various merge strategies including LLM-powered intelligent merging.+"""++from .base import BaseMerger, MergeStrategy+from .standard import KeepNewMerger, KeepOldMerger, FieldMerger+from .llm import LLMMerger++__all__ = [+ "BaseMerger",+ "MergeStrategy",+ "KeepNewMerger",+ "KeepOldMerger",+ "FieldMerger",+ "LLMMerger",+]diff --git a/hyperextract/utils/merger/base.py b/hyperextract/utils/merger/base.pynew file mode 100644index 0000000..d53d778--- /dev/null+++ b/hyperextract/utils/merger/base.py@@ -0,0 +1,347 @@+"""Base merger class implementing tournament-style merge algorithm.++The tournament merge algorithm efficiently handles n-way merging of duplicate items+by organizing merges into logarithmic rounds, similar to tournament elimination.+"""++import math+import logging+from abc import ABC, abstractmethod+from enum import Enum+from typing import List, Any, Callable, TypeVar, Generic, Dict, Tuple, Optional+from collections import defaultdict++try:+ from hyperextract.config import logger+except ImportError:+ logger = logging.getLogger(__name__)++T = TypeVar("T") # Generic item type+++class MergeStrategy(str, Enum):+ """Merge strategy for handling duplicate items.++ This enum defines the standard strategies for merging duplicate items+ when the same unique key is encountered multiple times.++ Strategies:+ KEEP_OLD: Keep the existing item, discard the new one+ KEEP_NEW: Keep the new item, discard the existing one (default)+ FIELD_MERGE: Merge fields from both items (new fills old's None fields)+ LLM_MERGE: Use LLM to intelligently merge both items++ Example:+ >>> from hyperextract.utils.merger import MergeStrategy+ >>> strategy = MergeStrategy.FIELD_MERGE+ >>> print(strategy.value) # "field_merge"+ """++ KEEP_OLD = "keep_old"+ KEEP_NEW = "keep_new"+ FIELD_MERGE = "field_merge"+ LLM_MERGE = "llm_merge"+++class BaseMerger(ABC, Generic[T]):+ """Abstract base class for item mergers with tournament-style merge algorithm.++ This class provides the core tournament merge algorithm that efficiently handles+ n duplicate items by organizing merges into O(log n) rounds. Each round collects+ all merge tasks across ALL unique keys and processes them in a single batch,+ maximizing efficiency for batch operations (e.g., LLM API calls).++ Key Features:+ - Tournament algorithm: O(log n) rounds for n duplicate items+ - Batch optimization: Each round processes ALL keys' pairs together+ - Automatic deduplication: Based on key_extractor function+ - Pluggable strategies: Subclasses implement pair_merge() and optionally batch_merge()++ Subclass Requirements:+ - Must implement: pair_merge(existing, incoming) -> merged_item+ - Optional override: batch_merge(pairs) -> merged_items (for batch optimization)++ Algorithm Example (5 duplicate items for one key):+ Initial: [i1, i2, i3, i4, i5]++ Round 1: Pair up [(i1,i2), (i3,i4)], advance [i5]+ → batch_merge([(i1,i2), (i3,i4)]) → [m12, m34]+ → Next round: [m12, m34, i5]++ Round 2: Pair up [(m12,m34)], advance [i5]+ → batch_merge([(m12,m34)]) → [m1234]+ → Next round: [m1234, i5]++ Round 3: Pair up [(m1234,i5)]+ → batch_merge([(m1234,i5)]) → [final]++ Total: 3 rounds, 4 merge operations (all batchable)++ Batch Optimization Example:+ - 10 unique keys, each with 4 duplicate items (2 pairs per key)+ - Without batch: 10 keys × 2 pairs = 20 separate API calls+ - With batch: Round 1 collects all 20 pairs → 1 API call+ - Speedup: 20x reduction in API calls++ Example:+ >>> class SimpleKeepNewMerger(BaseMerger[Item]):+ ... def pair_merge(self, existing, incoming):+ ... return incoming+ >>>+ >>> merger = SimpleKeepNewMerger(key_extractor=lambda x: x.id)+ >>> merged_items = merger.merge(items)+ """++ def __init__(+ self,+ key_extractor: Callable[[T], Any],+ *,+ logger_instance: Optional[logging.Logger] = None,+ ):+ """Initialize the merger.++ Args:+ key_extractor: Function to extract unique key from an item.+ Example: lambda x: x.id or lambda x: x.name+ logger_instance: Optional logger instance for progress tracking.+ If None, uses module-level logger.+ """+ self.key_extractor = key_extractor+ self.logger = logger_instance or logger++ # ==================== Abstract Methods ====================++ @abstractmethod+ def pair_merge(self, existing: T, incoming: T) -> T:+ """Merge two items (must be implemented by subclasses).++ This method defines how to merge a pair of duplicate items.++ Args:+ existing: The existing item.+ incoming: The incoming item to merge with existing.++ Returns:+ The merged item.++ Example:+ >>> def pair_merge(self, existing, incoming):+ ... # Keep the newer item+ ... return incoming+ """+ pass++ def batch_merge(self, pairs: List[Tuple[T, T]]) -> List[T]:+ """Batch merge multiple pairs (optional override for optimization).++ Default implementation: Sequentially call pair_merge() for each pair.++ Subclasses should override this method to implement true batch optimization,+ such as batching LLM API calls to reduce latency and cost.++ Args:+ pairs: List of (existing, incoming) tuples to merge.++ Returns:+ List of merged items (same order as input pairs).++ Example (LLM optimization):+ >>> def batch_merge(self, pairs):+ ... # Prepare batch inputs+ ... inputs = [{"a": a.json(), "b": b.json()} for a, b in pairs]+ ... # Single batch API call+ ... return llm_chain.batch(inputs)+ """+ results = []+ for existing, incoming in pairs:+ merged = self.pair_merge(existing, incoming)+ results.append(merged)+ return results++ # ==================== Core Algorithm ====================++ def merge(self, items: List[T]) -> List[T]:+ """Main entry point: Deduplicate and merge items with cross-key batching.++ Algorithm:+ 1. Group items by unique key+ 2. Run tournament merge across ALL keys simultaneously+ 3. Each round collects pairs from ALL keys → single batch_merge() call+ 4. Return list of merged items++ This cross-key batching significantly improves efficiency for batch operations+ (e.g., LLM API calls), reducing total API calls by orders of magnitude.++ Args:+ items: List of items to merge (may contain duplicates).++ Returns:+ List of deduplicated and merged items.++ Example:+ >>> items = [+ ... Item(id=1, name="Alice", age=None),+ ... Item(id=1, name=None, age=30),+ ... Item(id=2, name="Bob", age=25),+ ... Item(id=2, name="Bob", city="NYC"),+ ... ]+ >>> merger = FieldMerger(key_extractor=lambda x: x.id)+ >>> merged = merger.merge(items)+ >>> # Round 1: Batch merge [(Alice1, Alice2), (Bob1, Bob2)] in ONE call+ >>> # Result: [Item(id=1, name="Alice", age=30), Item(id=2, name="Bob", city="NYC")]+ """+ if not items:+ return []++ # Step 1: Group items by unique key+ groups = self._group_by_key(items)++ self.logger.info(f"Grouped {len(items)} items into {len(groups)} unique keys")++ # Step 2: Cross-key tournament merge+ merged_items = self._cross_key_tournament_merge(groups)++ self.logger.info(f"Merged into {len(merged_items)} unique items")++ return merged_items++ def _group_by_key(self, items: List[T]) -> Dict[Any, List[T]]:+ """Group items by their unique key.++ Args:+ items: List of items to group.++ Returns:+ Dictionary mapping key -> list of items with that key.++ Example:+ >>> items = [Item(id=1, ...), Item(id=1, ...), Item(id=2, ...)]+ >>> groups = self._group_by_key(items)+ >>> # Result: {1: [Item(id=1, ...), Item(id=1, ...)], 2: [Item(id=2, ...)]}+ """+ groups = defaultdict(list)+ for item in items:+ try:+ key = self.key_extractor(item)+ if key is not None:+ groups[key].append(item)+ else:+ self.logger.warning(f"Item has None key, skipping: {item}")+ except Exception as e:+ self.logger.warning(f"Failed to extract key from item: {e}")+ return dict(groups)++ def _cross_key_tournament_merge(self, groups: Dict[Any, List[T]]) -> List[T]:+ """Cross-key tournament merge: Process all keys simultaneously.++ **Key Optimization**: Instead of merging each key independently, this method+ runs tournament rounds across ALL keys, collecting all pairs from all keys+ in each round and processing them in a single batch_merge() call.++ Performance Impact:+ - Old approach: K keys × log(M) rounds × 1 batch call = K×log(M) API calls+ - New approach: log(max_M) rounds × 1 batch call = log(max_M) API calls+ - Speedup: K× reduction in API calls++ Algorithm:+ 1. Initialize each key with its list of items+ 2. Round 1: For each key, pair up items → collect ALL pairs → batch merge+ 3. Round 2: For each key, pair up results → collect ALL pairs → batch merge+ 4. Continue until each key has only 1 item++ Args:+ groups: Dictionary mapping key -> list of items with that key.++ Returns:+ List of merged items (one per unique key).++ Example (2 keys, 4 items each):+ Initial:+ key1: [i1a, i1b, i1c, i1d]+ key2: [i2a, i2b, i2c, i2d]++ Round 1: Collect ALL pairs from ALL keys+ key1 pairs: [(i1a,i1b), (i1c,i1d)]+ key2 pairs: [(i2a,i2b), (i2c,i2d)]+ ALL pairs: [(i1a,i1b), (i1c,i1d), (i2a,i2b), (i2c,i2d)]+ batch_merge(ALL 4 pairs) → [m1ab, m1cd, m2ab, m2cd]+ Next round:+ key1: [m1ab, m1cd]+ key2: [m2ab, m2cd]++ Round 2: Collect ALL pairs from ALL keys+ key1 pairs: [(m1ab,m1cd)]+ key2 pairs: [(m2ab,m2cd)]+ ALL pairs: [(m1ab,m1cd), (m2ab,m2cd)]+ batch_merge(ALL 2 pairs) → [final1, final2]++ Result: [final1, final2]+ Total API calls: 2 (vs 4 in old approach)+ """+ # Track current round of items for each key+ key_rounds: Dict[Any, List[T]] = {+ key: items[:] for key, items in groups.items()+ }++ round_num = 0+ max_rounds = max(+ math.ceil(math.log2(len(items))) if len(items) > 1 else 0+ for items in groups.values()+ )++ self.logger.debug(+ f"Starting cross-key tournament merge for {len(groups)} keys "+ f"(max {max_rounds} rounds)"+ )++ # Continue until all keys have exactly 1 item+ while any(len(items) > 1 for items in key_rounds.values()):+ round_num += 1+ all_pairs = [] # Collect pairs from ALL keys+ pair_to_key = [] # Track which key each pair belongs to++ # For each key, pair up items in current round+ for key, current_items in key_rounds.items():+ if len(current_items) <= 1:+ continue # This key is done++ # Pair up: (0,1), (2,3), (4,5), ...+ for i in range(0, len(current_items) - 1, 2):+ all_pairs.append((current_items[i], current_items[i + 1]))+ pair_to_key.append(key)++ # Batch merge ALL pairs from ALL keys in ONE call+ if all_pairs:+ self.logger.debug(+ f"Round {round_num}/{max_rounds}: Batch merging {len(all_pairs)} pairs "+ f"from {len([k for k, items in key_rounds.items() if len(items) > 1])} keys"+ )++ merged_results = self.batch_merge(all_pairs)++ # Distribute results back to keys+ result_idx = 0+ for key in key_rounds.keys():+ if len(key_rounds[key]) <= 1:+ continue++ next_round = []++ # Add merged results for this key's pairs+ key_pair_count = sum(1 for k in pair_to_key if k == key)+ for _ in range(key_pair_count):+ next_round.append(merged_results[result_idx])+ result_idx += 1++ # Add odd item if exists+ if len(key_rounds[key]) % 2 == 1:+ next_round.append(key_rounds[key][-1])++ key_rounds[key] = next_round++ # Extract final merged item for each key+ merged_items = [items[0] for items in key_rounds.values()]++ self.logger.debug(f"Cross-key tournament merge completed in {round_num} rounds")++ return merged_itemsdiff --git a/hyperextract/utils/merger/llm.py b/hyperextract/utils/merger/llm.pynew file mode 100644index 0000000..f92e51d--- /dev/null+++ b/hyperextract/utils/merger/llm.py@@ -0,0 +1,240 @@+"""LLM-powered intelligent merger with batch optimization.++Provides LLMMerger class that uses language models to intelligently merge+duplicate items. Key features:+- Batch API calls for efficiency (10-100x speedup)+- Structured output via Pydantic models+- Customizable merge prompts+- Automatic fallback on errors+"""++import logging+from typing import List, Tuple, Type, TypeVar, Optional+from pydantic import BaseModel+from langchain_core.language_models.chat_models import BaseChatModel+from langchain_core.prompts import ChatPromptTemplate++from .base import BaseMerger++try:+ from hyperextract.config import logger+except ImportError:+ logger = logging.getLogger(__name__)++T = TypeVar("T", bound=BaseModel)+++class LLMMerger(BaseMerger[T]):+ """LLM-powered intelligent merger with batch optimization.++ This merger uses a language model to intelligently merge duplicate items+ by understanding the semantic content and making smart decisions about+ how to combine information from both items.++ Key Features:+ - Batch optimization: Overrides batch_merge() to minimize API calls+ - Structured output: Returns Pydantic models matching input schema+ - Custom prompts: Supports user-defined merge instructions+ - Error handling: Automatic fallback to keep_new strategy++ Performance:+ - Without batching: N duplicate items = N API calls+ - With batching: N duplicate items = O(log N) API calls+ - Typical speedup: 10-100x reduction in API calls++ Requirements:+ - Items must be Pydantic BaseModel instances+ - LLM must support structured output++ Example:+ >>> from pydantic import BaseModel+ >>> from langchain_openai import ChatOpenAI+ >>>+ >>> class Person(BaseModel):+ ... name: str+ ... title: str | None = None+ ... skills: list[str] | None = None+ >>>+ >>> llm = ChatOpenAI(model="gpt-4")+ >>> merger = LLMMerger(+ ... key_extractor=lambda x: x.name,+ ... llm_client=llm,+ ... item_schema=Person,+ ... )+ >>>+ >>> items = [+ ... Person(name="孙悟空", title="齐天大圣", skills=None),+ ... Person(name="孙悟空", title=None, skills=["七十二变"]),+ ... Person(name="孙悟空", title=None, skills=["筋斗云"]),+ ... ]+ >>>+ >>> merged = merger.merge(items)+ >>> # LLM intelligently merges: Person(name="孙悟空", title="齐天大圣",+ >>> # skills=["七十二变", "筋斗云"])+ """++ def __init__(+ self,+ key_extractor,+ llm_client: BaseChatModel,+ item_schema: Type[T],+ *,+ merge_prompt_template: Optional[str] = None,+ logger_instance: Optional[logging.Logger] = None,+ ):+ """Initialize LLM merger.++ Args:+ key_extractor: Function to extract unique key from item.+ llm_client: LangChain chat model instance (must support structured output).+ item_schema: Pydantic model class defining item structure.+ merge_prompt_template: Custom prompt template for merging.+ If None, uses default template.+ logger_instance: Optional logger for progress tracking.++ Example:+ >>> merger = LLMMerger(+ ... key_extractor=lambda x: x.id,+ ... llm_client=ChatOpenAI(model="gpt-4"),+ ... item_schema=MyItemSchema,+ ... merge_prompt_template="Merge these items: {item_existing} and {item_incoming}",+ ... )+ """+ super().__init__(key_extractor, logger_instance=logger_instance)++ self.llm_client = llm_client+ self.item_schema = item_schema++ # Create merge chain with structured output+ template = merge_prompt_template or self._default_merge_template()+ merge_prompt = ChatPromptTemplate.from_template(template)+ self.merge_chain = merge_prompt | llm_client.with_structured_output(item_schema)++ @staticmethod+ def _default_merge_template() -> str:+ """Default merge prompt template.++ Returns:+ Prompt template string with {item_existing} and {item_incoming} placeholders.+ """+ return """You are an expert at merging structured data intelligently.++Given two instances of the same item (identified by the same unique key),+your task is to merge their fields into one complete, accurate item.++**Merging Rules:**+1. Keep the unique key field unchanged (it's the same in both items)+2. For other fields:+ - If only one item has a non-null value, use that value+ - If both items have values, choose the more complete/accurate one+ - For list fields, combine unique elements from both+ - For text fields, merge or choose the more informative one+3. Preserve all valuable information from both items+4. Return a single merged item that represents the best combination++**Item A (existing):**+{item_existing}++**Item B (incoming):**+{item_incoming}++**Instructions:**+Analyze both items carefully and return a single merged item that intelligently+combines information from both sources. Ensure the output matches the expected schema."""++ def pair_merge(self, existing: T, incoming: T) -> T:+ """Merge a single pair using LLM (fallback method).++ This method is used as a fallback when batch_merge fails or when+ called directly. For best performance, batch_merge() should be used.++ Args:+ existing: The existing item.+ incoming: The incoming item.++ Returns:+ Merged item from LLM, or incoming item if LLM fails.+ """+ try:+ self.logger.debug("Performing single LLM merge (fallback)")+ merged = self.merge_chain.invoke(+ {+ "item_existing": existing.model_dump_json(indent=2),+ "item_incoming": incoming.model_dump_json(indent=2),+ }+ )+ return merged+ except Exception as e:+ self.logger.error(+ f"LLM pair merge failed: {e}. Falling back to keep_new strategy."+ )+ return incoming++ def batch_merge(self, pairs: List[Tuple[T, T]]) -> List[T]:+ """Batch merge multiple pairs using LLM (optimized).++ **This is the key optimization**: Instead of making N separate API calls,+ we batch all pairs into a single API call, reducing latency and cost+ by 10-100x.++ Tournament merge algorithm calls this method once per round, with ALL+ pairs from all unique keys in that round. This maximizes batch size+ and minimizes API calls.++ Args:+ pairs: List of (existing, incoming) tuples to merge.++ Returns:+ List of merged items (same order as input).++ Example:+ >>> # Round 1 of tournament merge across 10 keys+ >>> # Each key has 4 items → 2 pairs per key → 20 pairs total+ >>> pairs = [+ ... (item1a, item1b), # Key 1, pair 1+ ... (item1c, item1d), # Key 1, pair 2+ ... (item2a, item2b), # Key 2, pair 1+ ... # ... 17 more pairs from other keys+ ... ]+ >>> # Single batch API call processes all 20 pairs at once+ >>> merged = merger.batch_merge(pairs)+ >>> # Result: 20 merged items in one API call (vs 20 separate calls)+ """+ if not pairs:+ return []++ self.logger.info(+ f"Batch merging {len(pairs)} pairs with LLM "+ f"(single API call instead of {len(pairs)} calls)"+ )++ # Prepare batch inputs+ inputs = [+ {+ "item_existing": existing.model_dump_json(indent=2),+ "item_incoming": incoming.model_dump_json(indent=2),+ }+ for existing, incoming in pairs+ ]++ try:+ # Critical: Single batch API call for all pairs+ merged_results = self.merge_chain.batch(inputs)++ self.logger.info(f"Successfully batch merged {len(merged_results)} pairs")++ return merged_results++ except Exception as e:+ self.logger.error(+ f"Batch LLM merge failed: {e}. "+ f"Falling back to sequential pair_merge for {len(pairs)} pairs."+ )++ # Fallback: Sequential pair merges+ results = []+ for existing, incoming in pairs:+ merged = self.pair_merge(existing, incoming)+ results.append(merged)++ return resultsdiff --git a/hyperextract/utils/merger/standard.py b/hyperextract/utils/merger/standard.pynew file mode 100644index 0000000..494c83b--- /dev/null+++ b/hyperextract/utils/merger/standard.py@@ -0,0 +1,131 @@+"""Standard merge strategies for common use cases.++Provides ready-to-use merger implementations for typical merge scenarios:+- KeepNewMerger: Always keep the incoming item+- KeepOldMerger: Always keep the existing item+- FieldMerger: Merge at field level (Pydantic models)+"""++import logging+from typing import List, TypeVar+from .base import BaseMerger++try:+ from hyperextract.config import logger+except ImportError:+ logger = logging.getLogger(__name__)++T = TypeVar("T")+++class KeepNewMerger(BaseMerger[T]):+ """Merger that always keeps the incoming (newer) item.++ Use case: When you want the most recent version of duplicate items.++ Example:+ >>> merger = KeepNewMerger(key_extractor=lambda x: x.id)+ >>> items = [Item(id=1, version=1), Item(id=1, version=2)]+ >>> merged = merger.merge(items)+ >>> # Result: [Item(id=1, version=2)]+ """++ def pair_merge(self, existing: T, incoming: T) -> T:+ """Keep the incoming item, discard the existing one.++ Args:+ existing: The existing item (will be discarded).+ incoming: The incoming item (will be kept).++ Returns:+ The incoming item.+ """+ return incoming+++class KeepOldMerger(BaseMerger[T]):+ """Merger that always keeps the existing (older) item.++ Use case: When you want to preserve the first occurrence of duplicate items.++ Example:+ >>> merger = KeepOldMerger(key_extractor=lambda x: x.id)+ >>> items = [Item(id=1, version=1), Item(id=1, version=2)]+ >>> merged = merger.merge(items)+ >>> # Result: [Item(id=1, version=1)]+ """++ def pair_merge(self, existing: T, incoming: T) -> T:+ """Keep the existing item, discard the incoming one.++ Args:+ existing: The existing item (will be kept).+ incoming: The incoming item (will be discarded).++ Returns:+ The existing item.+ """+ return existing+++class FieldMerger(BaseMerger[T]):+ """Merger that combines fields from both items (Pydantic models).++ Merge strategy:+ 1. Start with incoming item as base+ 2. Fill in None fields with existing item's non-None values+ 3. Result: Most complete item with fields from both++ Requirements:+ - Items must be Pydantic BaseModel instances+ - Items must have compatible schemas++ Use case: Accumulating information across multiple extractions.++ Example:+ >>> from pydantic import BaseModel+ >>> class Person(BaseModel):+ ... name: str+ ... age: int | None = None+ ... city: str | None = None+ >>>+ >>> merger = FieldMerger(key_extractor=lambda x: x.name)+ >>> items = [+ ... Person(name="Alice", age=30, city=None),+ ... Person(name="Alice", age=None, city="NYC"),+ ... ]+ >>> merged = merger.merge(items)+ >>> # Result: [Person(name="Alice", age=30, city="NYC")]+ """++ def pair_merge(self, existing: T, incoming: T) -> T:+ """Merge fields from both items, preferring non-None values.++ Strategy:+ - Start with incoming item+ - For each field in existing that is not None:+ - If incoming's field is None, use existing's value+ - Otherwise, keep incoming's value++ Args:+ existing: The existing item.+ incoming: The incoming item.++ Returns:+ Merged item with fields from both.++ Raises:+ AttributeError: If items are not Pydantic models.+ """+ try:+ # Use Pydantic's model_copy with update+ # existing.model_dump(exclude_none=True) gives all non-None fields+ # These values will override incoming's None fields+ return incoming.model_copy(update=existing.model_dump(exclude_none=True))+ except AttributeError as e:+ logger.error(+ f"FieldMerger requires Pydantic BaseModel instances. "+ f"Got types: {type(existing)}, {type(incoming)}. Error: {e}"+ )+ # Fallback to keep incoming+ return incoming