Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
重构日志配置,新增日志记录功能并统一日志导入方式;删除冗余的配置文件
yifanfeng9706ccccf89ed1
diff --git a/hyperextract/config.py b/hyperextract/config.pydeleted file mode 100644index e5d6112..0000000--- a/hyperextract/config.py+++ /dev/null@@ -1,36 +0,0 @@-# hyperextract/config.py-import sys-import os-from pathlib import Path-from dotenv import load_dotenv-from loguru import logger--# 1. Auto load .env-# Find the .env file in the project root directory-env_path = Path(__file__).parent.parent / ".env"-load_dotenv(dotenv_path=env_path)--# 2. Configure Loguru-# Remove default handler (to avoid duplication or inconsistent formatting)-logger.remove()--# Add console output (development mode level can be DEBUG)-LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")-logger.add(- sys.stderr,- level=LOG_LEVEL,- format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",-)--# Add file output (automatic rotation, save log files)-logger.add(- "logs/hyperextract.log",- rotation="10 MB", # Split file when it exceeds 10MB- retention="10 days", # Only keep recent 10 days- level="DEBUG", # Record more detailed information in file- compression="zip", # Compress historical logs-)---# Export logger for use in other modules-__all__ = ["logger"]\ No newline at end of filediff --git a/hyperextract/core/base.py b/hyperextract/core/base.pyindex 77842db..8748216 100644--- a/hyperextract/core/base.py+++ b/hyperextract/core/base.py@@ -72,15 +72,15 @@ class BaseAutoType(ABC, Generic[T]):separators=["\n\n", "\n", "。", "!", "?", ". ", "! ", "? ", " ", ""],)+ # Initialize internal state (calls hook for subclass setup)+ self._init_internal_state()+# Internal state storing the extracted knowledgeself.metadata: Dict[str, Any] = {"created_at": datetime.now(),"updated_at": datetime.now(),}- # Initialize internal state (calls hook for subclass setup)- self._init_internal_state()-def _create_empty_instance(self) -> "BaseAutoType[T]":"""Creates a new empty instance with the same configuration as this one.@@ -351,116 +351,34 @@ class BaseAutoType(ABC, Generic[T]):"""pass- # ==================== Serialization: Core Interface ====================-- def dump(self, folder_path: str | Path) -> None:- """Exports knowledge to a specified folder.-- Storage Structure:- folder/- ├── config.json # Metadata, Schema Info, Parameters- ├── data.json # Pure Knowledge Data- └── index/ # Vector Index Files+ # ==================== Serialization: Public Interface ====================+ def dump_data(self, file_path: str | Path) -> None:+ """Saves the pure knowledge data to disk as JSON.+Args:- folder_path: Target folder path for saving.+ file_path: Target file path for saving data (e.g., "data.json")."""- folder = Path(folder_path)- if folder.is_file():- raise FileExistsError(f"Path is a file, expected directory: {folder}")- folder.mkdir(parents=True, exist_ok=True)-- # 1. Save Configuration (Schema, Metadata, Parameters)- self._save_config(folder)-- # 2. Save Data (Pure Data)- self._save_data(folder)-- # 3. Save Index (Optional)- if self._index is not None:- self._dump_index_storage(folder)-- def load(self, folder_path: str | Path) -> None:- """Loads knowledge from a specified folder.-- Args:- folder_path: Source folder path containing saved knowledge.- """- folder = Path(folder_path)- if not folder.is_dir():- raise FileNotFoundError(f"Folder not found: {folder_path}")-- # 1. Load Configuration (Validate compatibility)- self._load_config(folder)-- # 2. Load Data (Restore state)- self._load_data(folder)-- # 3. Load Index (Restore vector search)- self._load_index_storage(folder)-- # ==================== Serialization: Components ====================-- def _save_config(self, folder: Path) -> None:- """Saves metadata, schema info, and instance parameters."""- config = {- "version": "1.0",- "type": self.__class__.__name__,- "schema": self._data_schema.__name__,- "params": {- "prompt": self.prompt,- "chunk_size": self.chunk_size,- "chunk_overlap": self.chunk_overlap,- "max_workers": self.max_workers,- },- "metadata": {- k: str(v) if isinstance(v, datetime) else v- for k, v in self.metadata.items()- },- "extra": self._get_extra_config_data(),- }-- with open(folder / "config.json", "w", encoding="utf-8") as f:- json.dump(config, f, indent=2, ensure_ascii=False)-- def _save_data(self, folder: Path) -> None:- """Saves the pure knowledge data."""+ path = Path(file_path)+ # Ensure parent directory exists+ path.parent.mkdir(parents=True, exist_ok=True)+export_data = self.data.model_dump()- with open(folder / "data.json", "w", encoding="utf-8") as f:+ with open(path, "w", encoding="utf-8") as f:json.dump(export_data, f, indent=2, ensure_ascii=False, default=str)- def _load_config(self, folder: Path) -> None:- """Loads and validates configuration."""- config_path = folder / "config.json"- if not config_path.exists():- raise FileNotFoundError(f"Config file not found: {config_path}")-- with open(config_path, "r", encoding="utf-8") as f:- config = json.load(f)-- # Strict schema validation - raise error if mismatch- if config.get("schema") != self._data_schema.__name__:- raise ValueError(- f"Schema mismatch: Expected '{self._data_schema.__name__}', "- f"but found '{config.get('schema')}' in config.json"- )-- # Restore metadata- if "metadata" in config:- self.metadata.update(config["metadata"])-- # Restore extra config- if "extra" in config:- self._load_extra_config_data(config["extra"])-- def _load_data(self, folder: Path) -> None:- """Loads data and restores internal state."""- data_path = folder / "data.json"- if not data_path.exists():- raise FileNotFoundError(f"Data file not found: {data_path}")+ def load_data(self, file_path: str | Path) -> None:+ """Loads data from disk and restores internal state.++ Args:+ file_path: Source file path containing data (e.g., "data.json").+ """+ path = Path(file_path)+ if not path.exists():+ raise FileNotFoundError(f"Data file not found: {path}")- with open(data_path, "r", encoding="utf-8") as f:+ with open(path, "r", encoding="utf-8") as f:raw_data = json.load(f)# Validate and Set State@@ -468,48 +386,26 @@ class BaseAutoType(ABC, Generic[T]):self._set_data_state(validated_data)@abstractmethod- def _dump_index_storage(self, folder: Path) -> None:- """HOOK: Save vector index to disk.+ def dump_index(self, folder_path: str | Path) -> None:+ """Save vector index to disk.Subclasses must implement to support their specific vector store(FAISS, Chroma, Pinecone, etc.).Args:- folder: Target folder for saving index files.+ folder_path: Target folder path for saving index files."""pass@abstractmethod- def _load_index_storage(self, folder: Path) -> None:- """HOOK: Load vector index from disk.+ def load_index(self, folder_path: str | Path) -> None:+ """Load vector index from disk.Subclasses must implement to support their specific vector store(FAISS, Chroma, Pinecone, etc.).Args:- folder: Source folder containing index files.- """- pass-- # ==================== Serialization: Hooks ====================-- def _get_extra_config_data(self) -> Dict[str, Any]:- """Hook to save subclass-specific configuration (not data).-- Override this method to save subclass-specific config like merge strategy.-- Returns:- Dictionary of extra config data.- """- return {}-- def _load_extra_config_data(self, extra_config: Dict[str, Any]) -> None:- """Hook to load subclass-specific configuration.-- Override this method to restore subclass-specific config.-- Args:- extra_config: Extra configuration data from config.json.+ folder_path: Source folder path containing index files."""passdiff --git a/hyperextract/core/list.py b/hyperextract/core/list.pyindex a3df171..8ec45e2 100644--- a/hyperextract/core/list.py+++ b/hyperextract/core/list.py@@ -2,32 +2,26 @@from pathlib import Pathfrom typing import (- List,Any,+ List,Type,+ Union,TypeVar,Generic,- TYPE_CHECKING,Iterator,Callable,Iterable,- Union,+ TYPE_CHECKING,)from datetime import datetimefrom pydantic import BaseModel, Field, create_model-from langchain_core.language_models.chat_models import BaseChatModel-from langchain_core.embeddings import Embeddingsfrom langchain_core.documents import Document+from langchain_core.embeddings import Embeddings+from langchain_core.language_models.chat_models import BaseChatModelfrom langchain_community.vectorstores import FAISSfrom .base import BaseAutoType--try:- from hyperextract.config import logger-except ImportError:- import logging-- logger = logging.getLogger(__name__)+from hyperextract.utils.logging import loggerItem = TypeVar("Item", bound=BaseModel)@@ -98,6 +92,14 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):),)+ # check fields_for_index validity+ if self.fields_for_index:+ for field_name in self.fields_for_index:+ if field_name not in item_schema.model_fields:+ raise ValueError(+ f"Field '{field_name}' not found in item schema '{item_schema.__name__}'"+ )+super().__init__(data_schema=self.item_list_schema,llm_client=llm_client,@@ -210,7 +212,7 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):def build_index(self) -> None:"""Builds independent vector index for each item in the list.-+If fields_for_index is specified, only those fields are indexed.Otherwise, all fields are indexed."""@@ -228,12 +230,14 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):if self.fields_for_index:# Only index specified fieldsitem_dict = item.model_dump()- indexed_fields = {k: v for k, v in item_dict.items() if k in self.fields_for_index}+ indexed_fields = {+ k: v for k, v in item_dict.items() if k in self.fields_for_index+ }content = str(indexed_fields)else:# Index all fieldscontent = item.model_dump_json()-+documents.append(Document(page_content=content,@@ -244,7 +248,9 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):if documents:try:self._index = FAISS.from_documents(documents, self.embedder)- logger.info(f"Built FAISS index with {len(documents)} items (fields: {self.fields_for_index or 'all'})")+ logger.info(+ f"Built FAISS index with {len(documents)} items (fields: {self.fields_for_index or 'all'})"+ )except ImportError:logger.error("FAISS not available. Install with: pip install faiss-cpu")raise@@ -283,17 +289,18 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):# ==================== Index Storage ====================- def _dump_index_storage(self, folder: Path) -> None:+ def dump_index(self, folder_path: str | Path) -> None:"""Saves FAISS vector index to disk."""if self._index is None:return+ folder = Path(folder_path)+ folder.mkdir(parents=True, exist_ok=True)index_path = str(folder / "index")self._index.save_local(index_path)- def _load_index_storage(self, folder: Path) -> None:+ def load_index(self, folder_path: str | Path) -> None:"""Loads FAISS vector index from disk."""- from langchain_community.vectorstores import FAISS-+ folder = Path(folder_path)index_path = folder / "index"if index_path.exists():try:diff --git a/hyperextract/core/model.py b/hyperextract/core/model.pyindex 17fcdf6..88ffde7 100644--- a/hyperextract/core/model.py+++ b/hyperextract/core/model.py@@ -11,7 +11,7 @@ from langchain_community.vectorstores import FAISSfrom .base import BaseAutoType, Ttry:- from hyperextract.config import logger+ from hyperextract.utils.logging import loggerexcept ImportError:import logging@@ -168,18 +168,6 @@ class AutoModel(BaseAutoType[T]):self._index = FAISS.from_documents(documents, self.embedder)logger.info(f"Built FAISS index with {len(documents)} documents")- def __len__(self) -> int:- """Returns 1 if there is data (non-empty fields), 0 otherwise.-- AutoModel always represents a single unit of knowledge, so it's either- empty (0) or contains one item (1).- """- if not self._data:- return 0- # Check if any field is non-null- has_data = any(v is not None for v in self._data.model_dump().values())- return 1 if has_data else 0-def search(self, query: str, top_k: int = 3) -> List[Any]:"""Searches all indexed fields using semantic similarity.@@ -214,17 +202,20 @@ class AutoModel(BaseAutoType[T]):# ==================== Index Storage ====================- def _dump_index_storage(self, folder: Path) -> None:+ def dump_index(self, folder_path: str | Path) -> None:"""Saves FAISS vector index to disk."""if self._index is None:return+ folder = Path(folder_path)+ folder.mkdir(parents=True, exist_ok=True)index_path = str(folder / "index")self._index.save_local(index_path)- def _load_index_storage(self, folder: Path) -> None:+ def load_index(self, folder_path: str | Path) -> None:"""Loads FAISS vector index from disk."""from langchain_community.vectorstores import FAISS+ folder = Path(folder_path)index_path = folder / "index"if index_path.exists():try:diff --git a/hyperextract/core/set.py b/hyperextract/core/set.pyindex 686aec8..37ece35 100644--- a/hyperextract/core/set.py+++ b/hyperextract/core/set.py@@ -6,14 +6,14 @@ Supports multiple merge strategies including LLM-powered intelligent merging.from pathlib import Pathfrom typing import (- List,Any,+ List,Type,+ Union,TypeVar,Generic,Optional,Callable,- Union,TYPE_CHECKING,)from datetime import datetime@@ -26,7 +26,7 @@ from langchain_core.documents import Documentfrom langchain_community.vectorstores import FAISSfrom .base import BaseAutoType-from hyperextract.config import logger+from hyperextract.utils.logging import loggerItem = TypeVar("Item", bound=BaseModel)@@ -48,7 +48,14 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):Key characteristics:- Extraction target: A unique collection of structured objects- Deduplication: Based on key_extractor function (user-specified)- - Merge strategy: Configurable (keep_old/keep_new/field_merge/llm_merge)+ - Merge strategy: Configurable via MergeStrategy enum:+ * KEEP_EXISTING: Preserve first (original) data, ignore updates+ * KEEP_INCOMING: Always use latest data, overwrite existing+ * MERGE_FIELD: Non-null fields overwrite, lists append (default)+ * LLM.BALANCED: LLM intelligently synthesizes both versions+ * LLM.PREFER_EXISTING: LLM synthesis but prioritizes original data+ * LLM.PREFER_INCOMING: LLM synthesis but prioritizes new data+ * LLM.CUSTOM_RULE: User-defined rules with dynamic context- Internal storage: Dict for O(1) lookup and deduplication- External interface: List (via items property)- Set operations: union (|), intersection (&), difference (-)@@ -84,17 +91,16 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):item_schema: Type[Item],llm_client: BaseChatModel,embedder: Embeddings,- *,key_extractor: Callable[[Item], Any],+ *,strategy_or_merger: Union[MergeStrategy, BaseMerger- ] = MergeStrategy.LLM.PREFER_INCOMING,+ ] = MergeStrategy.LLM.BALANCED,prompt: str = "",chunk_size: int = 2000,chunk_overlap: int = 200,max_workers: int = 10,show_progress: bool = True,- llm_batch_size: int = 10,**kwargs: Any,):"""Initialize AutoSet with key extractor and merge strategy.@@ -112,7 +118,6 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):chunk_overlap: Overlapping characters between adjacent chunks.max_workers: Maximum concurrent extraction tasks.show_progress: Whether to log progress information.- llm_batch_size: Batch size for LLM merge operations (for LLM_MERGE strategy).**kwargs: Additional arguments passed to create_merger() when strategy_or_merger isa MergeStrategy enum. Ignored if strategy_or_merger is a BaseMerger instance."""@@ -133,7 +138,6 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):# AutoSet-specific attributes (MUST be initialized BEFORE super().__init__ calls _init_internal_state)self.key_extractor = key_extractorself.strategy_or_merger = strategy_or_merger- self.llm_batch_size = llm_batch_size# Setup Merge Strategyif isinstance(strategy_or_merger, BaseMerger):@@ -155,7 +159,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):)# Initialize OMem instance BEFORE calling super().__init__ so _init_internal_state can use it- self._data: OMem[Item] = OMem(+ self._data_memory: OMem[Item] = OMem(memory_schema=item_schema,key_extractor=key_extractor,llm_client=llm_client,@@ -196,7 +200,6 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):chunk_overlap=self.chunk_overlap,max_workers=self.max_workers,show_progress=self.show_progress,- llm_batch_size=self.llm_batch_size,)def _default_prompt(self) -> str:@@ -216,7 +219,10 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):Returns:The internal knowledge data as a Pydantic model instance."""- return self.data_schema(items=self.items)+ if len(self.data.items) != len(self.items):+ # Sync data from OMem+ self._data = self.data_schema(items=self.items)+ return self._data@propertydef items(self) -> List[Item]:@@ -225,7 +231,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):Returns:List of unique items."""- return self._data.items+ return self._data_memory.items@propertydef keys(self) -> List[Any]:@@ -234,43 +240,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):Returns:List of unique key values."""- return self._data.keys-- def __len__(self) -> int:- """Returns the number of unique items in the set."""- return len(self._data.items)-- def __contains__(self, key: Any) -> bool:- """Checks if a unique key exists in the set.-- Args:- key: The unique key value to check.-- Returns:- True if key exists, False otherwise.- """- return self._data.get(key) is not None-- def __repr__(self) -> str:- """Returns a developer-friendly representation."""- return f"AutoSet[{self.item_schema.__name__}]({len(self)} unique items)"-- def __str__(self) -> str:- """Returns a user-friendly string representation."""- return f"AutoSet with {len(self)} unique {self.item_schema.__name__} items"-- def __iter__(self):- """Enables iteration over all items in the set.-- Returns:- Iterator over all unique items.-- Examples:- >>> for skill in skills:- ... print(skill.name)- >>> names = [s.name for s in skills]- """- return iter(self.items)+ return self._data_memory.keys# ==================== State Management Lifecycle Overrides ====================@@ -279,20 +249,21 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):INIT/RESET: Initialize or reset OMem as empty.Called during __init__ and when clear() is called."""- self._data.clear()+ self._data = self.item_set_schema()+ self._data_memory.clear()def _init_index_state(self) -> None:"""Initialize vector index to empty state."""- self._index = None+ self._data_memory.clear_index()def _set_data_state(self, data: AutoSetSchema[Item]) -> None:"""SET: Full Reset. Wipe OMem and refill from data (e.g., load from disk).Called by extract() or load() where data IS the new state."""- self._data.clear()+ self._data_memory.clear()if data.items:- self._data.add(data.items)+ self._data_memory.add(data.items)self.clear_index()def _update_data_state(self, incoming_data: AutoSetSchema[Item]) -> None:@@ -304,7 +275,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):handles deduplication and merging internally."""if incoming_data.items:- self._data.add(incoming_data.items)+ self._data_memory.add(incoming_data.items)self.clear_index()# ==================== Core Override Methods ====================@@ -336,57 +307,10 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):)merged_items = self._merger.merge(all_items)-- # Get strategy value for logging- if isinstance(self.strategy_or_merger, BaseMerger):- strategy_str = (- self.strategy_or_merger.strategy- if hasattr(self.strategy_or_merger, "strategy")- else "custom_merger"- )- else:- strategy_str = (- self.strategy_or_merger.value- if hasattr(self.strategy_or_merger, "value")- else str(self.strategy_or_merger)- )-- logger.info(- f"Merged into {len(merged_items)} unique items (strategy: {strategy_str})"- )+ logger.info(f"Merged into {len(merged_items)} unique items.")return self.item_set_schema(items=merged_items)- # ==================== Serialization Helpers ====================-- def _get_extra_dump_data(self) -> dict:- """Add AutoSet specific data to the dump.-- Returns:- Dictionary with extra AutoSet-specific fields.- """- # Get strategy value: handle different types of strategy_or_merger- if isinstance(self.strategy_or_merger, BaseMerger):- strategy_value = (- self.strategy_or_merger.strategy- if hasattr(self.strategy_or_merger, "strategy")- else "custom_merger"- )- elif isinstance(self.strategy_or_merger, str):- strategy_value = self.strategy_or_merger- else:- # MergeStrategy enum- strategy_value = (- self.strategy_or_merger.value- if hasattr(self.strategy_or_merger, "value")- else str(self.strategy_or_merger)- )-- return {- "item_schema_name": self.item_schema.__name__,- "strategy_or_merger": strategy_value,- }-# ==================== Indexing & Query ====================def build_index(self) -> None:@@ -452,17 +376,20 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):# ==================== Index Storage ====================- def _dump_index_storage(self, folder: Path) -> None:+ def dump_index(self, folder_path: str | Path) -> None:"""Saves FAISS vector index to disk."""if self._index is None:return+ folder = Path(folder_path)+ folder.mkdir(parents=True, exist_ok=True)index_path = str(folder / "index")self._index.save_local(index_path)- def _load_index_storage(self, folder: Path) -> None:+ def load_index(self, folder_path: str | Path) -> None:"""Loads FAISS vector index from disk."""from langchain_community.vectorstores import FAISS+ folder = Path(folder_path)index_path = folder / "index"if index_path.exists():try:@@ -474,6 +401,42 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):else:self._index = None+ def __len__(self) -> int:+ """Returns the number of unique items in the set."""+ return len(self._data_memory.items)++ def __contains__(self, key: Any) -> bool:+ """Checks if a unique key exists in the set.++ Args:+ key: The unique key value to check.++ Returns:+ True if key exists, False otherwise.+ """+ return self._data_memory.get(key) is not None++ def __repr__(self) -> str:+ """Returns a developer-friendly representation."""+ return f"AutoSet[{self.item_schema.__name__}]({len(self)} unique items)"++ def __str__(self) -> str:+ """Returns a user-friendly string representation."""+ return f"AutoSet with {len(self)} unique {self.item_schema.__name__} items"++ def __iter__(self):+ """Enables iteration over all items in the set.++ Returns:+ Iterator over all unique items.++ Examples:+ >>> for skill in skills:+ ... print(skill.name)+ >>> names = [s.name for s in skills]+ """+ return iter(self.items)+# ==================== Set-Specific Methods ====================def add(self, item: Item) -> None:@@ -483,7 +446,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):item: The item to add."""# OMem handles deduplication and merging automatically- self._data.add([item])+ self._data_memory.add([item])self.clear_index()self.metadata["updated_at"] = datetime.now()@@ -497,12 +460,12 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):The removed item, or None if not found."""# Get item before removing- item = self._data.get(key)+ item = self._data_memory.get(key)if item is None:return None# Remove from OMem- removed = self._data.remove(key)+ removed = self._data_memory.remove(key)if removed:self.clear_index()self.metadata["updated_at"] = datetime.now()@@ -518,7 +481,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):Returns:True if key exists, False otherwise."""- return self._data.get(key) is not None+ return self._data_memory.get(key) is not Nonedef get(self, key: Any, default: Optional[Item] = None) -> Optional[Item]:"""Gets an item by its unique key value.@@ -530,7 +493,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):Returns:The item if found, otherwise default."""- result = self._data.get(key)+ result = self._data_memory.get(key)return result if result is not None else defaultdef update(self, items: List[Item]) -> None:@@ -558,8 +521,8 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):- Updates metadata timestamp"""try:- if self._data.get(key) is not None:- self._data.remove(key)+ if self._data_memory.get(key) is not None:+ self._data_memory.remove(key)self.clear_index()self.metadata["updated_at"] = datetime.now()except (KeyError, Exception):@@ -585,12 +548,12 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):- Clears the vector index (needs rebuild)- Updates metadata timestamp"""- if not self._data.items:+ if not self._data_memory.items:raise KeyError("pop from an empty AutoSet")- item = self._data.items[0]+ item = self._data_memory.items[0]key = self.key_extractor(item)- self._data.remove(key)+ self._data_memory.remove(key)self.clear_index()self.metadata["updated_at"] = datetime.now()@@ -611,8 +574,8 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):new_set = self._create_empty_instance()# Add copies of all items- items_copy = [item.model_copy(deep=True) for item in self._data.items]- new_set._data.add(items_copy)+ items_copy = [item.model_copy(deep=True) for item in self._data_memory.items]+ new_set._data_memory.add(items_copy)new_set.metadata = self.metadata.copy()new_set.metadata["created_at"] = datetime.now()@@ -646,8 +609,8 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):new_set = self._create_empty_instance()# Add all items from both sets (OMem handles merge automatically)- all_items = self._data.items + other._data.items- new_set._data.add(all_items)+ all_items = self._data_memory.items + other._data_memory.items+ new_set._data_memory.add(all_items)new_set.metadata["updated_at"] = datetime.now()return new_set@@ -679,11 +642,13 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):# Only keep items present in both sets (by key)intersection_items = [- item for item in self._data.items if self.key_extractor(item) in other.keys+ item+ for item in self._data_memory.items+ if self.key_extractor(item) in other.keys]if intersection_items:- new_set._data.add(intersection_items)+ new_set._data_memory.add(intersection_items)new_set.metadata["updated_at"] = datetime.now()@@ -717,12 +682,12 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):# Only keep items not in other (by key)difference_items = [item- for item in self._data.items+ for item in self._data_memory.itemsif self.key_extractor(item) not in other.keys]if difference_items:- new_set._data.add(difference_items)+ new_set._data_memory.add(difference_items)new_set.metadata["updated_at"] = datetime.now()@@ -756,7 +721,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):# Items only in selfsymmetric_items = [item- for item in self._data.items+ for item in self._data_memory.itemsif self.key_extractor(item) not in other.keys]@@ -764,13 +729,13 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):symmetric_items.extend([item- for item in other._data.items+ for item in other._data_memory.itemsif self.key_extractor(item) not in self.keys])if symmetric_items:- new_set._data.add(symmetric_items)+ new_set._data_memory.add(symmetric_items)new_set.metadata["updated_at"] = datetime.now()diff --git a/hyperextract/utils/__init__.py b/hyperextract/utils/__init__.pyindex 596ffa7..8e8a47b 100644--- a/hyperextract/utils/__init__.py+++ b/hyperextract/utils/__init__.py@@ -8,6 +8,7 @@ from .merger import (FieldMerger,LLMMerger,)+from .logging import logger, setup_logger__all__ = ["BaseMerger",@@ -16,4 +17,6 @@ __all__ = ["KeepOldMerger","FieldMerger","LLMMerger",+ "logger",+ "setup_logger",]diff --git a/hyperextract/utils/logging.py b/hyperextract/utils/logging.pynew file mode 100644index 0000000..d5c8e09--- /dev/null+++ b/hyperextract/utils/logging.py@@ -0,0 +1,101 @@+"""+Loguru Logger Configuration++Provides flexible logging setup with optional file output.+Supports environment variables for configuration without requiring code changes.+"""++import os+import sys+from pathlib import Path+from typing import Optional, Union+from loguru import logger++# ==================== Environment Variable Constants ====================+ENV_LOG_LEVEL = "HYPER_EXTRACT_LOG_LEVEL"+ENV_LOG_FILE = "HYPER_EXTRACT_LOG_FILE"+++def setup_logger(+ level: str = "INFO",+ log_file: Optional[Union[str, Path]] = None,+ rotation: str = "10 MB",+ retention: str = "10 days",+ serialize: bool = False,+) -> None:+ """+ Configure Loguru Logger with optional file output.++ This function can be called explicitly by users to override default configuration.+ Environment variables are checked first, then parameters, then defaults.++ Args:+ level: Log level (e.g., "DEBUG", "INFO", "WARNING", "ERROR").+ Defaults to "INFO". Overridden by HYPER_EXTRACT_LOG_LEVEL env var.+ log_file: Path to log file. If None, only console output is used.+ Defaults to None. Overridden by HYPER_EXTRACT_LOG_FILE env var.+ rotation: Log rotation size (e.g., "10 MB", "1 GB", "00:00").+ retention: Log retention duration (e.g., "10 days", "7 days").+ serialize: If True, outputs logs in JSON format (suitable for production).++ Examples:+ # Default: console only, INFO level+ setup_logger()++ # Add file logging via code+ setup_logger(level="DEBUG", log_file="logs/debug.log")++ # Using environment variables (recommended for deployment)+ # HYPER_EXTRACT_LOG_LEVEL=DEBUG+ # HYPER_EXTRACT_LOG_FILE=./logs/app.log+ """+ # Remove all existing handlers to prevent duplication+ logger.remove()++ # 1. Determine final configuration (environment variables have highest priority)+ final_level = os.getenv(ENV_LOG_LEVEL, level).upper()+ env_log_file = os.getenv(ENV_LOG_FILE)+ final_log_file = log_file if log_file else env_log_file++ # 2. Add console handler+ console_format = (+ "<green>{time:YYYY-MM-DD HH:mm:ss}</green> | "+ "<level>{level: <8}</level> | "+ "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - "+ "<level>{message}</level>"+ )++ logger.add(+ sys.stderr,+ level=final_level,+ format=console_format,+ serialize=serialize,+ colorize=True,+ )++ # 3. Add file handler (only if path is configured)+ if final_log_file:+ log_path = Path(final_log_file)+ # Ensure parent directory exists+ log_path.parent.mkdir(parents=True, exist_ok=True)++ logger.add(+ str(log_path),+ rotation=rotation,+ retention=retention,+ level="DEBUG", # File captures more detailed information+ compression="zip",+ encoding="utf-8",+ enqueue=True, # Asynchronous I/O for thread safety+ serialize=serialize,+ )+++# ==================== Default Initialization ====================+# Initialize with sensible defaults on module import.+# This ensures users don't need explicit setup_logger() calls for basic usage.+# Environment variables will be checked and applied automatically.+setup_logger(level="INFO")++# Export public API+__all__ = ["logger", "setup_logger"]diff --git a/hyperextract/utils/merger/base.py b/hyperextract/utils/merger/base.pyindex d53d778..aee28fd 100644--- a/hyperextract/utils/merger/base.py+++ b/hyperextract/utils/merger/base.py@@ -12,7 +12,7 @@ from typing import List, Any, Callable, TypeVar, Generic, Dict, Tuple, Optionalfrom collections import defaultdicttry:- from hyperextract.config import logger+ from hyperextract.utils.logging import loggerexcept ImportError:logger = logging.getLogger(__name__)diff --git a/hyperextract/utils/merger/llm.py b/hyperextract/utils/merger/llm.pyindex f92e51d..2f3932f 100644--- a/hyperextract/utils/merger/llm.py+++ b/hyperextract/utils/merger/llm.py@@ -17,7 +17,7 @@ from langchain_core.prompts import ChatPromptTemplatefrom .base import BaseMergertry:- from hyperextract.config import logger+ from hyperextract.utils.logging import loggerexcept ImportError:logger = logging.getLogger(__name__)diff --git a/hyperextract/utils/merger/standard.py b/hyperextract/utils/merger/standard.pyindex 494c83b..feeeb05 100644--- a/hyperextract/utils/merger/standard.py+++ b/hyperextract/utils/merger/standard.py@@ -11,7 +11,7 @@ from typing import List, TypeVarfrom .base import BaseMergertry:- from hyperextract.config import logger+ from hyperextract.utils.logging import loggerexcept ImportError:logger = logging.getLogger(__name__)