Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

Add merger utilities with tournament-style algorithm and LLM integration

yifanfeng97d310ec6b030d
diff --git a/examples/set_knowledge_demo.py b/examples/set_knowledge_demo.py
index cf3255f..b372453 100644
--- a/examples/set_knowledge_demo.py
+++ b/examples/set_knowledge_demo.py
@@ -15,7 +15,8 @@ import os
from pydantic import BaseModel, Field
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
-from hyperextract.knowledge.generic.set import SetKnowledge, MergeItemStrategy
+from hyperextract.knowledge.generic.set import SetKnowledge
+from hyperextract.utils.merger import MergeStrategy
# Define the schema for skills
@@ -50,8 +51,8 @@ def main():
item_schema=SkillSchema,
llm_client=llm,
embedder=embedder,
- unique_key="name", # Deduplicate based on skill name
- merge_item_strategy=MergeItemStrategy.FIELD_MERGE,
+ key_extractor=lambda x: x.name, # Deduplicate based on skill name
+ merge_item_strategy=MergeStrategy.FIELD_MERGE,
prompt="""Extract all technical skills, frameworks, tools, and soft skills mentioned in the text.
For each skill, provide:
@@ -194,7 +195,7 @@ Be specific and extract all mentioned skills.""",
item_schema=SkillSchema,
llm_client=llm,
embedder=embedder,
- unique_key="name",
+ key_extractor=lambda x: x.name,
)
loaded_skills.load(save_path)
print(f"✓ Loaded {len(loaded_skills)} unique skills")
@@ -206,7 +207,7 @@ Be specific and extract all mentioned skills.""",
print(f" Total unique skills extracted: {len(skills)}")
print(f" Job descriptions processed: {len(job_descriptions)}")
print(" Deduplication strategy: FIELD_MERGE")
- print(" Unique key: 'name'")
+ print(" Key extractor: lambda x: x.name")
print("\n ✓ Automatically deduplicated based on skill names")
print(" ✓ Merged information from multiple occurrences")
print(" ✓ Supports semantic search across all skills")
diff --git a/hyperextract/knowledge/generic/__init__.py b/hyperextract/knowledge/generic/__init__.py
index 2e4df7a..79fca3e 100644
--- a/hyperextract/knowledge/generic/__init__.py
+++ b/hyperextract/knowledge/generic/__init__.py
@@ -2,12 +2,11 @@
from .unit import UnitKnowledge
from .list import ListKnowledge, ItemListSchema
-from .set import SetKnowledge, MergeItemStrategy
+from .set import SetKnowledge
__all__ = [
"UnitKnowledge",
"ListKnowledge",
"ItemListSchema",
"SetKnowledge",
- "MergeItemStrategy",
]
diff --git a/hyperextract/knowledge/generic/set.py b/hyperextract/knowledge/generic/set.py
index b381b6f..e551691 100644
--- a/hyperextract/knowledge/generic/set.py
+++ b/hyperextract/knowledge/generic/set.py
@@ -5,17 +5,23 @@ Supports multiple merge strategies including LLM-powered intelligent merging.
"""
import json
-from typing import List, Any, Type, TypeVar, Generic, Dict, Optional
+from typing import List, Any, Type, TypeVar, Generic, Dict, Optional, Callable
from datetime import datetime
from pathlib import Path
-from enum import Enum
from pydantic import BaseModel, Field, create_model
from langchain_core.language_models.chat_models import BaseChatModel
from langchain_core.embeddings import Embeddings
-from langchain_core.prompts import ChatPromptTemplate
from langchain_core.documents import Document
from ..base import BaseKnowledge
+from hyperextract.utils.merger import (
+ BaseMerger,
+ MergeStrategy,
+ KeepNewMerger,
+ KeepOldMerger,
+ FieldMerger,
+ LLMMerger,
+)
try:
from hyperextract.config import logger
@@ -34,32 +40,16 @@ class ItemSetSchema(BaseModel, Generic[Item]):
items: List[Item] = Field(default_factory=list, description="Set of unique items")
-class MergeItemStrategy(str, Enum):
- """Merge strategy for handling duplicate items.
-
- Strategies:
- KEEP_OLD: Keep the existing item, discard the new one
- KEEP_NEW: Keep the new item, discard the existing one (default)
- FIELD_MERGE: Merge fields from both items (new fills old's None fields)
- LLM_MERGE: Use LLM to intelligently merge both items
- """
-
- KEEP_OLD = "keep_old"
- KEEP_NEW = "keep_new"
- FIELD_MERGE = "field_merge"
- LLM_MERGE = "llm_merge"
-
-
class SetKnowledge(BaseKnowledge[ItemSetSchema[Item]], Generic[Item]):
"""Set Knowledge Pattern - extracts a unique collection of objects.
This pattern automatically deduplicates items based on a user-specified
- unique key field. Provides flexible merge strategies including LLM-powered
+ key extractor function. Provides flexible merge strategies including LLM-powered
intelligent merging for handling duplicates.
Key characteristics:
- Extraction target: A unique collection of structured objects
- - Deduplication: Based on unique_key field (user-specified)
+ - Deduplication: Based on key_extractor function (user-specified)
- Merge strategy: Configurable (keep_old/keep_new/field_merge/llm_merge)
- Internal storage: Dict for O(1) lookup and deduplication
- External interface: List (via items property)
@@ -79,7 +69,7 @@ class SetKnowledge(BaseKnowledge[ItemSetSchema[Item]], Generic[Item]):
... item_schema=KeywordSchema,
... llm_client=llm,
... embedder=embedder,
- ... unique_key="term",
+ ... key_extractor=lambda x: x.term,
... merge_item_strategy="field_merge"
... )
>>> keywords.extract("Python is great. Python is powerful.")
@@ -93,8 +83,8 @@ class SetKnowledge(BaseKnowledge[ItemSetSchema[Item]], Generic[Item]):
llm_client: BaseChatModel,
embedder: Embeddings,
*,
- unique_key: str,
- merge_item_strategy: MergeItemStrategy = MergeItemStrategy.KEEP_NEW,
+ key_extractor: Callable[[Item], Any],
+ merge_item_strategy: MergeStrategy = MergeStrategy.KEEP_NEW,
prompt: str = "",
chunk_size: int = 2000,
chunk_overlap: int = 200,
@@ -103,13 +93,13 @@ class SetKnowledge(BaseKnowledge[ItemSetSchema[Item]], Generic[Item]):
llm_batch_size: int = 10,
**kwargs,
):
- """Initialize SetKnowledge with unique key and merge strategy.
+ """Initialize SetKnowledge with key extractor and merge strategy.
Args:
item_schema: Pydantic BaseModel subclass for individual items.
llm_client: Language model client for extraction and merging.
embedder: Embedding model for vector indexing.
- unique_key: Field name to use as unique identifier (required).
+ key_extractor: Function to extract unique key from an item (required).
merge_item_strategy: Strategy for merging duplicate items (default: KEEP_NEW).
prompt: Custom extraction prompt.
chunk_size: Maximum characters per chunk for long texts.
@@ -118,13 +108,7 @@ class SetKnowledge(BaseKnowledge[ItemSetSchema[Item]], Generic[Item]):
show_progress: Whether to log progress information.
llm_batch_size: Batch size for LLM merge operations (for LLM_MERGE strategy).
**kwargs: Additional arguments passed to parent class.
-
- Raises:
- ValueError: If unique_key field does not exist in item_schema.
- TypeError: If unique_key field type is not hashable.
"""
- # Validate unique_key before initialization
- self._validate_unique_key(item_schema, unique_key)
# Store item_schema
self.item_schema = item_schema
@@ -153,18 +137,18 @@ class SetKnowledge(BaseKnowledge[ItemSetSchema[Item]], Generic[Item]):
)
# SetKnowledge-specific attributes
- self.unique_key = unique_key
+ self.key_extractor = key_extractor
self.merge_item_strategy = merge_item_strategy
self.llm_batch_size = llm_batch_size
# Internal storage: Dict for O(1) deduplication
self._items_dict: Dict[Any, Item] = {}
- # Create LLM merge chain (if using LLM_MERGE strategy)
- self._merge_chain = (
- self._create_merge_chain()
- if merge_item_strategy == MergeItemStrategy.LLM_MERGE
- else None
+ # Create Merger instance based on strategy
+ self._merger: BaseMerger[Item] = self._create_merger(
+ item_schema=item_schema,
+ llm_client=llm_client,
+ merge_item_strategy=merge_item_strategy,
)
# ==================== Override Instance Creation ====================
@@ -181,7 +165,7 @@ class SetKnowledge(BaseKnowledge[ItemSetSchema[Item]], Generic[Item]):
item_schema=self.item_schema,
llm_client=self.llm_client,
embedder=self.embedder,
- unique_key=self.unique_key,
+ key_extractor=self.key_extractor,
merge_item_strategy=self.merge_item_strategy,
prompt=self.prompt,
chunk_size=self.chunk_size,
@@ -204,78 +188,48 @@ class SetKnowledge(BaseKnowledge[ItemSetSchema[Item]], Generic[Item]):
"### Source Text:\n"
)
- @staticmethod
- def _validate_unique_key(item_schema: Type[Item], unique_key: str):
- """Validates that unique_key exists in schema and is hashable.
-
- Args:
- item_schema: The Pydantic schema to validate.
- unique_key: The field name to validate.
-
- Raises:
- ValueError: If unique_key field does not exist.
- TypeError: If unique_key field type is known to be unhashable.
- """
- # Check field existence
- if unique_key not in item_schema.model_fields:
- available_fields = list(item_schema.model_fields.keys())
- raise ValueError(
- f"unique_key '{unique_key}' not found in {item_schema.__name__}. "
- f"Available fields: {available_fields}"
- )
-
- # Check field type (basic unhashable type detection)
- field_info = item_schema.model_fields[unique_key]
- field_type = field_info.annotation
-
- # Extract actual type from Optional/Union if present
- import typing
-
- if (
- hasattr(typing, "get_origin")
- and typing.get_origin(field_type) is typing.Union
- ):
- args = typing.get_args(field_type)
- field_type = args[0] if args else field_type
-
- # Check for known unhashable types
- unhashable_types = (list, dict, set, List, Dict)
- if field_type in unhashable_types:
- raise TypeError(
- f"unique_key field '{unique_key}' has unhashable type {field_type}. "
- f"unique_key must be a hashable type (str, int, tuple, etc.)"
- )
-
- # ==================== LLM Merge Chain ====================
+ # ==================== Merger Factory ====================
- def _create_merge_chain(self):
- """Creates an LLM chain for intelligently merging duplicate items.
+ def _create_merger(
+ self,
+ item_schema: Type[Item],
+ llm_client: BaseChatModel,
+ merge_item_strategy: MergeStrategy,
+ ) -> BaseMerger[Item]:
+ """Creates the appropriate Merger instance based on strategy.
- Subclasses can override this method to customize the merge prompt.
+ Args:
+ item_schema: Pydantic model for items.
+ llm_client: LLM client for LLM_MERGE strategy.
+ merge_item_strategy: The merge strategy to use.
Returns:
- LangChain chain for merging items.
+ A Merger instance implementing the specified strategy.
"""
- merge_template = """You are an expert at merging structured data.
-Given two instances of the same item (identified by the same unique key),
-intelligently merge their fields into one complete item.
+ key_extractor = self.key_extractor
-Merging rules:
-1. Keep the unique key field unchanged
-2. For other fields: prefer non-null values
-3. If both items have values for a field, choose the more complete/accurate one
-4. Combine text fields if appropriate (e.g., descriptions)
+ if merge_item_strategy == MergeStrategy.KEEP_OLD:
+ return KeepOldMerger(key_extractor, logger_instance=logger)
-Item A (existing):
-{item_existing}
+ elif merge_item_strategy == MergeStrategy.KEEP_NEW:
+ return KeepNewMerger(key_extractor, logger_instance=logger)
-Item B (incoming):
-{item_incoming}
+ elif merge_item_strategy == MergeStrategy.FIELD_MERGE:
+ return FieldMerger(key_extractor, logger_instance=logger)
-Please merge these two items intelligently and return the merged result."""
+ elif merge_item_strategy == MergeStrategy.LLM_MERGE:
+ return LLMMerger(
+ key_extractor=key_extractor,
+ llm_client=llm_client,
+ item_schema=item_schema,
+ logger_instance=logger,
+ )
- merge_prompt = ChatPromptTemplate.from_template(merge_template)
- return merge_prompt | self.llm_client.with_structured_output(self.item_schema)
+ else:
+ logger.warning(
+ f"Unknown merge strategy: {merge_item_strategy}, using KEEP_NEW"
+ )
+ return KeepNewMerger(key_extractor, logger_instance=logger)
# ==================== Properties ====================
@@ -387,12 +341,10 @@ Please merge these two items intelligently and return the merged result."""
# If store=True, merge with existing data and update internal state
if store:
if self._data and len(self.items) > 0:
- self._data = self.merge([self._data, merged_data])
+ final_data = self.merge([self._data, merged_data])
else:
- self._data = merged_data
-
- self.clear_index()
- self.metadata["updated_at"] = datetime.now()
+ final_data = merged_data
+ self._update_internal_state(final_data)
logger.info("Knowledge extraction completed")
logger.info(
@@ -405,97 +357,61 @@ Please merge these two items intelligently and return the merged result."""
def merge(self, data_list: List[ItemSetSchema[Item]]) -> ItemSetSchema[Item]:
"""Merges multiple data containers with automatic deduplication.
- Overrides parent's merge to implement set semantics:
- 1. Collect all items from all containers
- 2. For each item, check if unique_key already exists
- 3. If duplicate, call merge_item() to resolve
- 4. Return deduplicated result
-
- For LLM_MERGE strategy, batches duplicate pairs for efficient processing.
+ Pure function: Does not modify internal state.
+ Delegates to Merger for efficient tournament-style merging.
+ All merge strategies (including LLM batch processing) are handled
+ by the Merger implementation.
Args:
data_list: List of container objects to merge.
Returns:
- Merged ItemSetSchema with deduplicated items.
+ New merged ItemSetSchema with deduplicated items.
"""
- # Collect all items and track duplicates for batch LLM merge
- merge_pairs = [] # [(existing, incoming, key), ...]
-
+ # Step 1: Collect all items from all containers
+ all_items = []
for data in data_list:
- for item in data.items:
- # Get unique key value
- key_value = self._get_key_value(item)
- if key_value is None:
- continue
-
- if key_value in self._items_dict:
- # Duplicate found
- if self.merge_item_strategy == MergeItemStrategy.LLM_MERGE:
- # Collect for batch processing
- merge_pairs.append(
- (self._items_dict[key_value], item, key_value)
- )
- else:
- # Merge immediately with non-LLM strategies
- merged = self.merge_item(self._items_dict[key_value], item)
- if merged is not None:
- self._items_dict[key_value] = merged
- else:
- # New item
- self._items_dict[key_value] = item.model_copy(deep=True)
-
- # Batch process LLM merges if any
- if merge_pairs and self.merge_item_strategy == MergeItemStrategy.LLM_MERGE:
- merged_results = self._batch_merge_with_llm(merge_pairs)
- # Update items_dict with merged results
- self._items_dict.update(merged_results)
-
- # Update internal _data to match _items_dict
- self._data.items = list(self._items_dict.values())
+ all_items.extend(data.items)
+
+ if not all_items:
+ return self.item_set_schema()
logger.info(
- f"Merged into {len(self._items_dict)} unique items "
+ f"Merging {len(all_items)} items from {len(data_list)} containers..."
+ )
+
+ # Step 2: Use Merger to deduplicate and merge items
+ merged_items = self._merger.merge(all_items)
+
+ logger.info(
+ f"Merged into {len(merged_items)} unique items "
f"(strategy: {self.merge_item_strategy.value})"
)
- return self._data
+ # Step 3: Return new container (does not modify self)
+ return self.item_set_schema(items=merged_items)
- def merge_item(self, existing: Item, incoming: Item) -> Optional[Item]:
- """Merges two duplicate items based on the configured strategy.
+ def _update_internal_state(self, data: ItemSetSchema[Item]) -> None:
+ """Updates internal state from a data container.
- This method can be overridden by subclasses to implement custom merge logic.
+ Private method that synchronizes _items_dict and _data with new data.
+ Also clears vector index and updates metadata timestamp.
Args:
- existing: The existing item in the set.
- incoming: The incoming item to merge.
-
- Returns:
- Merged item, or None to skip this item.
+ data: The data container to update from.
"""
- if self.merge_item_strategy == MergeItemStrategy.KEEP_OLD:
- return existing
-
- elif self.merge_item_strategy == MergeItemStrategy.KEEP_NEW:
- return incoming
-
- elif self.merge_item_strategy == MergeItemStrategy.FIELD_MERGE:
- # Field-level merge: existing's non-None values override incoming
- return incoming.model_copy(update=existing.model_dump(exclude_none=True))
-
- elif self.merge_item_strategy == MergeItemStrategy.LLM_MERGE:
- # LLM merge (should be handled by batch processing in merge())
- # Fallback to single merge if called directly
- return self._merge_item_with_llm(existing, incoming)
+ self._items_dict.clear()
+ for item in data.items:
+ key = self._get_key_value(item)
+ if key is not None:
+ self._items_dict[key] = item
- else:
- logger.warning(
- f"Unknown merge strategy: {self.merge_item_strategy}, using KEEP_NEW"
- )
- return incoming
+ self._data = data
+ self.clear_index()
+ self.metadata["updated_at"] = datetime.now()
def _get_key_value(self, item: Item) -> Optional[Any]:
- """Extracts the unique key value from an item.
+ """Extracts the unique key value from an item using key_extractor.
Args:
item: The item to extract key from.
@@ -504,114 +420,25 @@ Please merge these two items intelligently and return the merged result."""
The unique key value, or None if invalid.
"""
try:
- key_value = getattr(item, self.unique_key)
+ key_value = self.key_extractor(item)
# Validate key value
if key_value is None:
- logger.warning(
- f"Item has None value for unique_key '{self.unique_key}', skipping"
- )
+ logger.warning("Item has None value for key, skipping")
return None
# Verify hashability
hash(key_value)
return key_value
- except TypeError:
- logger.error(
- f"Value {key_value} for unique_key '{self.unique_key}' is not hashable"
- )
+ except TypeError as e:
+ logger.error(f"Key value is not hashable: {e}")
return None
- except AttributeError:
- logger.error(f"Item missing unique_key field '{self.unique_key}'")
+ except Exception as e:
+ logger.error(f"Failed to extract key from item: {e}")
return None
- # ==================== LLM Merge Methods ====================
-
- def _batch_merge_with_llm(self, merge_pairs: List[tuple]) -> Dict[Any, Item]:
- """Batch processes LLM merges for efficiency.
-
- Args:
- merge_pairs: List of (existing, incoming, key_value) tuples to merge.
-
- Returns:
- Dictionary mapping key_value to merged items.
- """
- results = {}
-
- if not self._merge_chain:
- logger.warning("LLM merge chain not initialized, falling back to KEEP_NEW")
- for existing, incoming, key_value in merge_pairs:
- results[key_value] = incoming
- return results
-
- total_pairs = len(merge_pairs)
- logger.info(f"Batch merging {total_pairs} duplicate items with LLM...")
-
- # Process in batches
- for i in range(0, total_pairs, self.llm_batch_size):
- batch = merge_pairs[i : i + self.llm_batch_size]
-
- # Prepare batch inputs
- inputs = [
- {
- "item_existing": existing.model_dump_json(indent=2),
- "item_incoming": incoming.model_dump_json(indent=2),
- }
- for existing, incoming, _ in batch
- ]
-
- try:
- # Batch invoke LLM
- merged_results = self._merge_chain.batch(inputs)
-
- # Collect merged results
- for (existing, incoming, key_value), merged in zip(
- batch, merged_results
- ):
- if merged:
- results[key_value] = merged
- logger.debug(f"Successfully merged item with key '{key_value}'")
- else:
- logger.warning(
- f"LLM merge returned None for key '{key_value}', keeping incoming"
- )
- results[key_value] = incoming
-
- except Exception as e:
- logger.error(f"Batch LLM merge failed: {e}, falling back to KEEP_NEW")
- # Fallback: keep incoming items
- for existing, incoming, key_value in batch:
- results[key_value] = incoming
-
- return results
-
- def _merge_item_with_llm(self, existing: Item, incoming: Item) -> Optional[Item]:
- """Uses LLM to intelligently merge two duplicate items (single merge).
-
- Args:
- existing: The existing item.
- incoming: The incoming item to merge.
-
- Returns:
- Merged item, or None on failure.
- """
- if not self._merge_chain:
- logger.warning("LLM merge chain not initialized")
- return incoming
-
- try:
- merged = self._merge_chain.invoke(
- {
- "item_existing": existing.model_dump_json(indent=2),
- "item_incoming": incoming.model_dump_json(indent=2),
- }
- )
- logger.debug("Successfully merged item with LLM")
- return merged
- except Exception as e:
- logger.error(f"LLM merge failed: {e}")
- return incoming # Fallback to incoming item
+ # ==================== LLM Merge Methods (Removed - Now handled by LLMMerger) ====================
def build_index(self):
"""Builds independent vector index for each unique item."""
@@ -694,7 +521,6 @@ Please merge these two items intelligently and return the merged result."""
"schema_name": self.item_set_schema.__name__,
"item_schema_name": self.item_schema.__name__,
"item_schema": self.item_schema.model_json_schema(),
- "unique_key": self.unique_key,
"merge_item_strategy": self.merge_item_strategy.value,
"data": self.data.model_dump(),
"metadata": {
@@ -736,16 +562,10 @@ Please merge these two items intelligently and return the merged result."""
data = json.load(f)
logger.info(f"Loaded data from {data_file}")
- self._data = self.item_set_schema.model_validate(data.get("data", {}))
+ loaded_data = self.item_set_schema.model_validate(data.get("data", {}))
+ self._update_internal_state(loaded_data)
- # Rebuild items_dict from _data.items (not self.items property!)
- self._items_dict = {}
- for item in self._data.items:
- key_value = self._get_key_value(item)
- if key_value is not None:
- self._items_dict[key_value] = item
-
- # Update metadata
+ # Update metadata from saved data
if "metadata" in data:
self.metadata.update(data["metadata"])
self.metadata["updated_at"] = datetime.now()
@@ -783,16 +603,17 @@ Please merge these two items intelligently and return the merged result."""
if key_value in self._items_dict:
# Merge with existing
- merged = self.merge_item(self._items_dict[key_value], item)
+ merged = self._merger.pair_merge(self._items_dict[key_value], item)
if merged:
self._items_dict[key_value] = merged
else:
# Add new
self._items_dict[key_value] = item.model_copy(deep=True)
- # Update _data
+ # Update internal state
self._data.items = list(self._items_dict.values())
self.clear_index()
+ self.metadata["updated_at"] = datetime.now()
def remove(self, key: Any) -> Optional[Item]:
"""Removes an item by its unique key value.
@@ -807,6 +628,7 @@ Please merge these two items intelligently and return the merged result."""
if removed:
self._data.items = list(self._items_dict.values())
self.clear_index()
+ self.metadata["updated_at"] = datetime.now()
logger.debug(f"Removed item with key '{key}'")
return removed
@@ -857,8 +679,8 @@ Please merge these two items intelligently and return the merged result."""
- Clears the vector index (needs rebuild)
- Updates metadata timestamp
"""
- removed = self._items_dict.pop(key, None)
- if removed:
+ if key in self._items_dict:
+ self._items_dict.pop(key)
self._data.items = list(self._items_dict.values())
self.clear_index()
self.metadata["updated_at"] = datetime.now()
@@ -946,7 +768,7 @@ Please merge these two items intelligently and return the merged result."""
# Add items from other (merge duplicates)
for key, item in other._items_dict.items():
if key in new_set._items_dict:
- merged = new_set.merge_item(new_set._items_dict[key], item)
+ merged = new_set._merger.pair_merge(new_set._items_dict[key], item)
if merged:
new_set._items_dict[key] = merged
else:
@@ -986,7 +808,7 @@ Please merge these two items intelligently and return the merged result."""
for key in self._items_dict:
if key in other._items_dict:
# Merge the two versions
- merged = new_set.merge_item(
+ merged = new_set._merger.pair_merge(
self._items_dict[key], other._items_dict[key]
)
if merged:
@@ -1095,7 +917,7 @@ Please merge these two items intelligently and return the merged result."""
def __eq__(self, other: Any) -> bool:
"""Equality comparison: set1 == set2.
- Two sets are equal if they have the same schema, unique_key, and key set.
+ Two sets are equal if they have the same schema and key set.
Note: Does not compare item contents, only keys.
Args:
@@ -1113,9 +935,6 @@ Please merge these two items intelligently and return the merged result."""
if self.item_schema != other.item_schema:
return False
- if self.unique_key != other.unique_key:
- return False
-
return set(self._items_dict.keys()) == set(other._items_dict.keys())
def __ne__(self, other: Any) -> bool:
diff --git a/hyperextract/utils/__init__.py b/hyperextract/utils/__init__.py
new file mode 100644
index 0000000..596ffa7
--- /dev/null
+++ b/hyperextract/utils/__init__.py
@@ -0,0 +1,19 @@
+"""Hyperextract utilities module."""
+
+from .merger import (
+ BaseMerger,
+ MergeStrategy,
+ KeepNewMerger,
+ KeepOldMerger,
+ FieldMerger,
+ LLMMerger,
+)
+
+__all__ = [
+ "BaseMerger",
+ "MergeStrategy",
+ "KeepNewMerger",
+ "KeepOldMerger",
+ "FieldMerger",
+ "LLMMerger",
+]
diff --git a/hyperextract/utils/merger/__init__.py b/hyperextract/utils/merger/__init__.py
new file mode 100644
index 0000000..9f3ba9d
--- /dev/null
+++ b/hyperextract/utils/merger/__init__.py
@@ -0,0 +1,18 @@
+"""Item merger utilities for knowledge patterns.
+
+Provides a flexible merger framework with tournament-style merge algorithm
+and support for various merge strategies including LLM-powered intelligent merging.
+"""
+
+from .base import BaseMerger, MergeStrategy
+from .standard import KeepNewMerger, KeepOldMerger, FieldMerger
+from .llm import LLMMerger
+
+__all__ = [
+ "BaseMerger",
+ "MergeStrategy",
+ "KeepNewMerger",
+ "KeepOldMerger",
+ "FieldMerger",
+ "LLMMerger",
+]
diff --git a/hyperextract/utils/merger/base.py b/hyperextract/utils/merger/base.py
new file mode 100644
index 0000000..d53d778
--- /dev/null
+++ b/hyperextract/utils/merger/base.py
@@ -0,0 +1,347 @@
+"""Base merger class implementing tournament-style merge algorithm.
+
+The tournament merge algorithm efficiently handles n-way merging of duplicate items
+by organizing merges into logarithmic rounds, similar to tournament elimination.
+"""
+
+import math
+import logging
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import List, Any, Callable, TypeVar, Generic, Dict, Tuple, Optional
+from collections import defaultdict
+
+try:
+ from hyperextract.config import logger
+except ImportError:
+ logger = logging.getLogger(__name__)
+
+T = TypeVar("T") # Generic item type
+
+
+class MergeStrategy(str, Enum):
+ """Merge strategy for handling duplicate items.
+
+ This enum defines the standard strategies for merging duplicate items
+ when the same unique key is encountered multiple times.
+
+ Strategies:
+ KEEP_OLD: Keep the existing item, discard the new one
+ KEEP_NEW: Keep the new item, discard the existing one (default)
+ FIELD_MERGE: Merge fields from both items (new fills old's None fields)
+ LLM_MERGE: Use LLM to intelligently merge both items
+
+ Example:
+ >>> from hyperextract.utils.merger import MergeStrategy
+ >>> strategy = MergeStrategy.FIELD_MERGE
+ >>> print(strategy.value) # "field_merge"
+ """
+
+ KEEP_OLD = "keep_old"
+ KEEP_NEW = "keep_new"
+ FIELD_MERGE = "field_merge"
+ LLM_MERGE = "llm_merge"
+
+
+class BaseMerger(ABC, Generic[T]):
+ """Abstract base class for item mergers with tournament-style merge algorithm.
+
+ This class provides the core tournament merge algorithm that efficiently handles
+ n duplicate items by organizing merges into O(log n) rounds. Each round collects
+ all merge tasks across ALL unique keys and processes them in a single batch,
+ maximizing efficiency for batch operations (e.g., LLM API calls).
+
+ Key Features:
+ - Tournament algorithm: O(log n) rounds for n duplicate items
+ - Batch optimization: Each round processes ALL keys' pairs together
+ - Automatic deduplication: Based on key_extractor function
+ - Pluggable strategies: Subclasses implement pair_merge() and optionally batch_merge()
+
+ Subclass Requirements:
+ - Must implement: pair_merge(existing, incoming) -> merged_item
+ - Optional override: batch_merge(pairs) -> merged_items (for batch optimization)
+
+ Algorithm Example (5 duplicate items for one key):
+ Initial: [i1, i2, i3, i4, i5]
+
+ Round 1: Pair up [(i1,i2), (i3,i4)], advance [i5]
+ → batch_merge([(i1,i2), (i3,i4)]) → [m12, m34]
+ → Next round: [m12, m34, i5]
+
+ Round 2: Pair up [(m12,m34)], advance [i5]
+ → batch_merge([(m12,m34)]) → [m1234]
+ → Next round: [m1234, i5]
+
+ Round 3: Pair up [(m1234,i5)]
+ → batch_merge([(m1234,i5)]) → [final]
+
+ Total: 3 rounds, 4 merge operations (all batchable)
+
+ Batch Optimization Example:
+ - 10 unique keys, each with 4 duplicate items (2 pairs per key)
+ - Without batch: 10 keys × 2 pairs = 20 separate API calls
+ - With batch: Round 1 collects all 20 pairs → 1 API call
+ - Speedup: 20x reduction in API calls
+
+ Example:
+ >>> class SimpleKeepNewMerger(BaseMerger[Item]):
+ ... def pair_merge(self, existing, incoming):
+ ... return incoming
+ >>>
+ >>> merger = SimpleKeepNewMerger(key_extractor=lambda x: x.id)
+ >>> merged_items = merger.merge(items)
+ """
+
+ def __init__(
+ self,
+ key_extractor: Callable[[T], Any],
+ *,
+ logger_instance: Optional[logging.Logger] = None,
+ ):
+ """Initialize the merger.
+
+ Args:
+ key_extractor: Function to extract unique key from an item.
+ Example: lambda x: x.id or lambda x: x.name
+ logger_instance: Optional logger instance for progress tracking.
+ If None, uses module-level logger.
+ """
+ self.key_extractor = key_extractor
+ self.logger = logger_instance or logger
+
+ # ==================== Abstract Methods ====================
+
+ @abstractmethod
+ def pair_merge(self, existing: T, incoming: T) -> T:
+ """Merge two items (must be implemented by subclasses).
+
+ This method defines how to merge a pair of duplicate items.
+
+ Args:
+ existing: The existing item.
+ incoming: The incoming item to merge with existing.
+
+ Returns:
+ The merged item.
+
+ Example:
+ >>> def pair_merge(self, existing, incoming):
+ ... # Keep the newer item
+ ... return incoming
+ """
+ pass
+
+ def batch_merge(self, pairs: List[Tuple[T, T]]) -> List[T]:
+ """Batch merge multiple pairs (optional override for optimization).
+
+ Default implementation: Sequentially call pair_merge() for each pair.
+
+ Subclasses should override this method to implement true batch optimization,
+ such as batching LLM API calls to reduce latency and cost.
+
+ Args:
+ pairs: List of (existing, incoming) tuples to merge.
+
+ Returns:
+ List of merged items (same order as input pairs).
+
+ Example (LLM optimization):
+ >>> def batch_merge(self, pairs):
+ ... # Prepare batch inputs
+ ... inputs = [{"a": a.json(), "b": b.json()} for a, b in pairs]
+ ... # Single batch API call
+ ... return llm_chain.batch(inputs)
+ """
+ results = []
+ for existing, incoming in pairs:
+ merged = self.pair_merge(existing, incoming)
+ results.append(merged)
+ return results
+
+ # ==================== Core Algorithm ====================
+
+ def merge(self, items: List[T]) -> List[T]:
+ """Main entry point: Deduplicate and merge items with cross-key batching.
+
+ Algorithm:
+ 1. Group items by unique key
+ 2. Run tournament merge across ALL keys simultaneously
+ 3. Each round collects pairs from ALL keys → single batch_merge() call
+ 4. Return list of merged items
+
+ This cross-key batching significantly improves efficiency for batch operations
+ (e.g., LLM API calls), reducing total API calls by orders of magnitude.
+
+ Args:
+ items: List of items to merge (may contain duplicates).
+
+ Returns:
+ List of deduplicated and merged items.
+
+ Example:
+ >>> items = [
+ ... Item(id=1, name="Alice", age=None),
+ ... Item(id=1, name=None, age=30),
+ ... Item(id=2, name="Bob", age=25),
+ ... Item(id=2, name="Bob", city="NYC"),
+ ... ]
+ >>> merger = FieldMerger(key_extractor=lambda x: x.id)
+ >>> merged = merger.merge(items)
+ >>> # Round 1: Batch merge [(Alice1, Alice2), (Bob1, Bob2)] in ONE call
+ >>> # Result: [Item(id=1, name="Alice", age=30), Item(id=2, name="Bob", city="NYC")]
+ """
+ if not items:
+ return []
+
+ # Step 1: Group items by unique key
+ groups = self._group_by_key(items)
+
+ self.logger.info(f"Grouped {len(items)} items into {len(groups)} unique keys")
+
+ # Step 2: Cross-key tournament merge
+ merged_items = self._cross_key_tournament_merge(groups)
+
+ self.logger.info(f"Merged into {len(merged_items)} unique items")
+
+ return merged_items
+
+ def _group_by_key(self, items: List[T]) -> Dict[Any, List[T]]:
+ """Group items by their unique key.
+
+ Args:
+ items: List of items to group.
+
+ Returns:
+ Dictionary mapping key -> list of items with that key.
+
+ Example:
+ >>> items = [Item(id=1, ...), Item(id=1, ...), Item(id=2, ...)]
+ >>> groups = self._group_by_key(items)
+ >>> # Result: {1: [Item(id=1, ...), Item(id=1, ...)], 2: [Item(id=2, ...)]}
+ """
+ groups = defaultdict(list)
+ for item in items:
+ try:
+ key = self.key_extractor(item)
+ if key is not None:
+ groups[key].append(item)
+ else:
+ self.logger.warning(f"Item has None key, skipping: {item}")
+ except Exception as e:
+ self.logger.warning(f"Failed to extract key from item: {e}")
+ return dict(groups)
+
+ def _cross_key_tournament_merge(self, groups: Dict[Any, List[T]]) -> List[T]:
+ """Cross-key tournament merge: Process all keys simultaneously.
+
+ **Key Optimization**: Instead of merging each key independently, this method
+ runs tournament rounds across ALL keys, collecting all pairs from all keys
+ in each round and processing them in a single batch_merge() call.
+
+ Performance Impact:
+ - Old approach: K keys × log(M) rounds × 1 batch call = K×log(M) API calls
+ - New approach: log(max_M) rounds × 1 batch call = log(max_M) API calls
+ - Speedup: K× reduction in API calls
+
+ Algorithm:
+ 1. Initialize each key with its list of items
+ 2. Round 1: For each key, pair up items → collect ALL pairs → batch merge
+ 3. Round 2: For each key, pair up results → collect ALL pairs → batch merge
+ 4. Continue until each key has only 1 item
+
+ Args:
+ groups: Dictionary mapping key -> list of items with that key.
+
+ Returns:
+ List of merged items (one per unique key).
+
+ Example (2 keys, 4 items each):
+ Initial:
+ key1: [i1a, i1b, i1c, i1d]
+ key2: [i2a, i2b, i2c, i2d]
+
+ Round 1: Collect ALL pairs from ALL keys
+ key1 pairs: [(i1a,i1b), (i1c,i1d)]
+ key2 pairs: [(i2a,i2b), (i2c,i2d)]
+ ALL pairs: [(i1a,i1b), (i1c,i1d), (i2a,i2b), (i2c,i2d)]
+ batch_merge(ALL 4 pairs) → [m1ab, m1cd, m2ab, m2cd]
+ Next round:
+ key1: [m1ab, m1cd]
+ key2: [m2ab, m2cd]
+
+ Round 2: Collect ALL pairs from ALL keys
+ key1 pairs: [(m1ab,m1cd)]
+ key2 pairs: [(m2ab,m2cd)]
+ ALL pairs: [(m1ab,m1cd), (m2ab,m2cd)]
+ batch_merge(ALL 2 pairs) → [final1, final2]
+
+ Result: [final1, final2]
+ Total API calls: 2 (vs 4 in old approach)
+ """
+ # Track current round of items for each key
+ key_rounds: Dict[Any, List[T]] = {
+ key: items[:] for key, items in groups.items()
+ }
+
+ round_num = 0
+ max_rounds = max(
+ math.ceil(math.log2(len(items))) if len(items) > 1 else 0
+ for items in groups.values()
+ )
+
+ self.logger.debug(
+ f"Starting cross-key tournament merge for {len(groups)} keys "
+ f"(max {max_rounds} rounds)"
+ )
+
+ # Continue until all keys have exactly 1 item
+ while any(len(items) > 1 for items in key_rounds.values()):
+ round_num += 1
+ all_pairs = [] # Collect pairs from ALL keys
+ pair_to_key = [] # Track which key each pair belongs to
+
+ # For each key, pair up items in current round
+ for key, current_items in key_rounds.items():
+ if len(current_items) <= 1:
+ continue # This key is done
+
+ # Pair up: (0,1), (2,3), (4,5), ...
+ for i in range(0, len(current_items) - 1, 2):
+ all_pairs.append((current_items[i], current_items[i + 1]))
+ pair_to_key.append(key)
+
+ # Batch merge ALL pairs from ALL keys in ONE call
+ if all_pairs:
+ self.logger.debug(
+ f"Round {round_num}/{max_rounds}: Batch merging {len(all_pairs)} pairs "
+ f"from {len([k for k, items in key_rounds.items() if len(items) > 1])} keys"
+ )
+
+ merged_results = self.batch_merge(all_pairs)
+
+ # Distribute results back to keys
+ result_idx = 0
+ for key in key_rounds.keys():
+ if len(key_rounds[key]) <= 1:
+ continue
+
+ next_round = []
+
+ # Add merged results for this key's pairs
+ key_pair_count = sum(1 for k in pair_to_key if k == key)
+ for _ in range(key_pair_count):
+ next_round.append(merged_results[result_idx])
+ result_idx += 1
+
+ # Add odd item if exists
+ if len(key_rounds[key]) % 2 == 1:
+ next_round.append(key_rounds[key][-1])
+
+ key_rounds[key] = next_round
+
+ # Extract final merged item for each key
+ merged_items = [items[0] for items in key_rounds.values()]
+
+ self.logger.debug(f"Cross-key tournament merge completed in {round_num} rounds")
+
+ return merged_items
diff --git a/hyperextract/utils/merger/llm.py b/hyperextract/utils/merger/llm.py
new file mode 100644
index 0000000..f92e51d
--- /dev/null
+++ b/hyperextract/utils/merger/llm.py
@@ -0,0 +1,240 @@
+"""LLM-powered intelligent merger with batch optimization.
+
+Provides LLMMerger class that uses language models to intelligently merge
+duplicate items. Key features:
+- Batch API calls for efficiency (10-100x speedup)
+- Structured output via Pydantic models
+- Customizable merge prompts
+- Automatic fallback on errors
+"""
+
+import logging
+from typing import List, Tuple, Type, TypeVar, Optional
+from pydantic import BaseModel
+from langchain_core.language_models.chat_models import BaseChatModel
+from langchain_core.prompts import ChatPromptTemplate
+
+from .base import BaseMerger
+
+try:
+ from hyperextract.config import logger
+except ImportError:
+ logger = logging.getLogger(__name__)
+
+T = TypeVar("T", bound=BaseModel)
+
+
+class LLMMerger(BaseMerger[T]):
+ """LLM-powered intelligent merger with batch optimization.
+
+ This merger uses a language model to intelligently merge duplicate items
+ by understanding the semantic content and making smart decisions about
+ how to combine information from both items.
+
+ Key Features:
+ - Batch optimization: Overrides batch_merge() to minimize API calls
+ - Structured output: Returns Pydantic models matching input schema
+ - Custom prompts: Supports user-defined merge instructions
+ - Error handling: Automatic fallback to keep_new strategy
+
+ Performance:
+ - Without batching: N duplicate items = N API calls
+ - With batching: N duplicate items = O(log N) API calls
+ - Typical speedup: 10-100x reduction in API calls
+
+ Requirements:
+ - Items must be Pydantic BaseModel instances
+ - LLM must support structured output
+
+ Example:
+ >>> from pydantic import BaseModel
+ >>> from langchain_openai import ChatOpenAI
+ >>>
+ >>> class Person(BaseModel):
+ ... name: str
+ ... title: str | None = None
+ ... skills: list[str] | None = None
+ >>>
+ >>> llm = ChatOpenAI(model="gpt-4")
+ >>> merger = LLMMerger(
+ ... key_extractor=lambda x: x.name,
+ ... llm_client=llm,
+ ... item_schema=Person,
+ ... )
+ >>>
+ >>> items = [
+ ... Person(name="孙悟空", title="齐天大圣", skills=None),
+ ... Person(name="孙悟空", title=None, skills=["七十二变"]),
+ ... Person(name="孙悟空", title=None, skills=["筋斗云"]),
+ ... ]
+ >>>
+ >>> merged = merger.merge(items)
+ >>> # LLM intelligently merges: Person(name="孙悟空", title="齐天大圣",
+ >>> # skills=["七十二变", "筋斗云"])
+ """
+
+ def __init__(
+ self,
+ key_extractor,
+ llm_client: BaseChatModel,
+ item_schema: Type[T],
+ *,
+ merge_prompt_template: Optional[str] = None,
+ logger_instance: Optional[logging.Logger] = None,
+ ):
+ """Initialize LLM merger.
+
+ Args:
+ key_extractor: Function to extract unique key from item.
+ llm_client: LangChain chat model instance (must support structured output).
+ item_schema: Pydantic model class defining item structure.
+ merge_prompt_template: Custom prompt template for merging.
+ If None, uses default template.
+ logger_instance: Optional logger for progress tracking.
+
+ Example:
+ >>> merger = LLMMerger(
+ ... key_extractor=lambda x: x.id,
+ ... llm_client=ChatOpenAI(model="gpt-4"),
+ ... item_schema=MyItemSchema,
+ ... merge_prompt_template="Merge these items: {item_existing} and {item_incoming}",
+ ... )
+ """
+ super().__init__(key_extractor, logger_instance=logger_instance)
+
+ self.llm_client = llm_client
+ self.item_schema = item_schema
+
+ # Create merge chain with structured output
+ template = merge_prompt_template or self._default_merge_template()
+ merge_prompt = ChatPromptTemplate.from_template(template)
+ self.merge_chain = merge_prompt | llm_client.with_structured_output(item_schema)
+
+ @staticmethod
+ def _default_merge_template() -> str:
+ """Default merge prompt template.
+
+ Returns:
+ Prompt template string with {item_existing} and {item_incoming} placeholders.
+ """
+ return """You are an expert at merging structured data intelligently.
+
+Given two instances of the same item (identified by the same unique key),
+your task is to merge their fields into one complete, accurate item.
+
+**Merging Rules:**
+1. Keep the unique key field unchanged (it's the same in both items)
+2. For other fields:
+ - If only one item has a non-null value, use that value
+ - If both items have values, choose the more complete/accurate one
+ - For list fields, combine unique elements from both
+ - For text fields, merge or choose the more informative one
+3. Preserve all valuable information from both items
+4. Return a single merged item that represents the best combination
+
+**Item A (existing):**
+{item_existing}
+
+**Item B (incoming):**
+{item_incoming}
+
+**Instructions:**
+Analyze both items carefully and return a single merged item that intelligently
+combines information from both sources. Ensure the output matches the expected schema."""
+
+ def pair_merge(self, existing: T, incoming: T) -> T:
+ """Merge a single pair using LLM (fallback method).
+
+ This method is used as a fallback when batch_merge fails or when
+ called directly. For best performance, batch_merge() should be used.
+
+ Args:
+ existing: The existing item.
+ incoming: The incoming item.
+
+ Returns:
+ Merged item from LLM, or incoming item if LLM fails.
+ """
+ try:
+ self.logger.debug("Performing single LLM merge (fallback)")
+ merged = self.merge_chain.invoke(
+ {
+ "item_existing": existing.model_dump_json(indent=2),
+ "item_incoming": incoming.model_dump_json(indent=2),
+ }
+ )
+ return merged
+ except Exception as e:
+ self.logger.error(
+ f"LLM pair merge failed: {e}. Falling back to keep_new strategy."
+ )
+ return incoming
+
+ def batch_merge(self, pairs: List[Tuple[T, T]]) -> List[T]:
+ """Batch merge multiple pairs using LLM (optimized).
+
+ **This is the key optimization**: Instead of making N separate API calls,
+ we batch all pairs into a single API call, reducing latency and cost
+ by 10-100x.
+
+ Tournament merge algorithm calls this method once per round, with ALL
+ pairs from all unique keys in that round. This maximizes batch size
+ and minimizes API calls.
+
+ Args:
+ pairs: List of (existing, incoming) tuples to merge.
+
+ Returns:
+ List of merged items (same order as input).
+
+ Example:
+ >>> # Round 1 of tournament merge across 10 keys
+ >>> # Each key has 4 items → 2 pairs per key → 20 pairs total
+ >>> pairs = [
+ ... (item1a, item1b), # Key 1, pair 1
+ ... (item1c, item1d), # Key 1, pair 2
+ ... (item2a, item2b), # Key 2, pair 1
+ ... # ... 17 more pairs from other keys
+ ... ]
+ >>> # Single batch API call processes all 20 pairs at once
+ >>> merged = merger.batch_merge(pairs)
+ >>> # Result: 20 merged items in one API call (vs 20 separate calls)
+ """
+ if not pairs:
+ return []
+
+ self.logger.info(
+ f"Batch merging {len(pairs)} pairs with LLM "
+ f"(single API call instead of {len(pairs)} calls)"
+ )
+
+ # Prepare batch inputs
+ inputs = [
+ {
+ "item_existing": existing.model_dump_json(indent=2),
+ "item_incoming": incoming.model_dump_json(indent=2),
+ }
+ for existing, incoming in pairs
+ ]
+
+ try:
+ # Critical: Single batch API call for all pairs
+ merged_results = self.merge_chain.batch(inputs)
+
+ self.logger.info(f"Successfully batch merged {len(merged_results)} pairs")
+
+ return merged_results
+
+ except Exception as e:
+ self.logger.error(
+ f"Batch LLM merge failed: {e}. "
+ f"Falling back to sequential pair_merge for {len(pairs)} pairs."
+ )
+
+ # Fallback: Sequential pair merges
+ results = []
+ for existing, incoming in pairs:
+ merged = self.pair_merge(existing, incoming)
+ results.append(merged)
+
+ return results
diff --git a/hyperextract/utils/merger/standard.py b/hyperextract/utils/merger/standard.py
new file mode 100644
index 0000000..494c83b
--- /dev/null
+++ b/hyperextract/utils/merger/standard.py
@@ -0,0 +1,131 @@
+"""Standard merge strategies for common use cases.
+
+Provides ready-to-use merger implementations for typical merge scenarios:
+- KeepNewMerger: Always keep the incoming item
+- KeepOldMerger: Always keep the existing item
+- FieldMerger: Merge at field level (Pydantic models)
+"""
+
+import logging
+from typing import List, TypeVar
+from .base import BaseMerger
+
+try:
+ from hyperextract.config import logger
+except ImportError:
+ logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+
+class KeepNewMerger(BaseMerger[T]):
+ """Merger that always keeps the incoming (newer) item.
+
+ Use case: When you want the most recent version of duplicate items.
+
+ Example:
+ >>> merger = KeepNewMerger(key_extractor=lambda x: x.id)
+ >>> items = [Item(id=1, version=1), Item(id=1, version=2)]
+ >>> merged = merger.merge(items)
+ >>> # Result: [Item(id=1, version=2)]
+ """
+
+ def pair_merge(self, existing: T, incoming: T) -> T:
+ """Keep the incoming item, discard the existing one.
+
+ Args:
+ existing: The existing item (will be discarded).
+ incoming: The incoming item (will be kept).
+
+ Returns:
+ The incoming item.
+ """
+ return incoming
+
+
+class KeepOldMerger(BaseMerger[T]):
+ """Merger that always keeps the existing (older) item.
+
+ Use case: When you want to preserve the first occurrence of duplicate items.
+
+ Example:
+ >>> merger = KeepOldMerger(key_extractor=lambda x: x.id)
+ >>> items = [Item(id=1, version=1), Item(id=1, version=2)]
+ >>> merged = merger.merge(items)
+ >>> # Result: [Item(id=1, version=1)]
+ """
+
+ def pair_merge(self, existing: T, incoming: T) -> T:
+ """Keep the existing item, discard the incoming one.
+
+ Args:
+ existing: The existing item (will be kept).
+ incoming: The incoming item (will be discarded).
+
+ Returns:
+ The existing item.
+ """
+ return existing
+
+
+class FieldMerger(BaseMerger[T]):
+ """Merger that combines fields from both items (Pydantic models).
+
+ Merge strategy:
+ 1. Start with incoming item as base
+ 2. Fill in None fields with existing item's non-None values
+ 3. Result: Most complete item with fields from both
+
+ Requirements:
+ - Items must be Pydantic BaseModel instances
+ - Items must have compatible schemas
+
+ Use case: Accumulating information across multiple extractions.
+
+ Example:
+ >>> from pydantic import BaseModel
+ >>> class Person(BaseModel):
+ ... name: str
+ ... age: int | None = None
+ ... city: str | None = None
+ >>>
+ >>> merger = FieldMerger(key_extractor=lambda x: x.name)
+ >>> items = [
+ ... Person(name="Alice", age=30, city=None),
+ ... Person(name="Alice", age=None, city="NYC"),
+ ... ]
+ >>> merged = merger.merge(items)
+ >>> # Result: [Person(name="Alice", age=30, city="NYC")]
+ """
+
+ def pair_merge(self, existing: T, incoming: T) -> T:
+ """Merge fields from both items, preferring non-None values.
+
+ Strategy:
+ - Start with incoming item
+ - For each field in existing that is not None:
+ - If incoming's field is None, use existing's value
+ - Otherwise, keep incoming's value
+
+ Args:
+ existing: The existing item.
+ incoming: The incoming item.
+
+ Returns:
+ Merged item with fields from both.
+
+ Raises:
+ AttributeError: If items are not Pydantic models.
+ """
+ try:
+ # Use Pydantic's model_copy with update
+ # existing.model_dump(exclude_none=True) gives all non-None fields
+ # These values will override incoming's None fields
+ return incoming.model_copy(update=existing.model_dump(exclude_none=True))
+ except AttributeError as e:
+ logger.error(
+ f"FieldMerger requires Pydantic BaseModel instances. "
+ f"Got types: {type(existing)}, {type(incoming)}. Error: {e}"
+ )
+ # Fallback to keep incoming
+ return incoming