Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

重构日志配置,新增日志记录功能并统一日志导入方式;删除冗余的配置文件

yifanfeng9706ccccf89ed1
diff --git a/hyperextract/config.py b/hyperextract/config.py
deleted file mode 100644
index e5d6112..0000000
--- a/hyperextract/config.py
+++ /dev/null
@@ -1,36 +0,0 @@
-# hyperextract/config.py
-import sys
-import os
-from pathlib import Path
-from dotenv import load_dotenv
-from loguru import logger
-
-# 1. Auto load .env
-# Find the .env file in the project root directory
-env_path = Path(__file__).parent.parent / ".env"
-load_dotenv(dotenv_path=env_path)
-
-# 2. Configure Loguru
-# Remove default handler (to avoid duplication or inconsistent formatting)
-logger.remove()
-
-# Add console output (development mode level can be DEBUG)
-LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
-logger.add(
- sys.stderr,
- level=LOG_LEVEL,
- format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
-)
-
-# Add file output (automatic rotation, save log files)
-logger.add(
- "logs/hyperextract.log",
- rotation="10 MB", # Split file when it exceeds 10MB
- retention="10 days", # Only keep recent 10 days
- level="DEBUG", # Record more detailed information in file
- compression="zip", # Compress historical logs
-)
-
-
-# Export logger for use in other modules
-__all__ = ["logger"]
\ No newline at end of file
diff --git a/hyperextract/core/base.py b/hyperextract/core/base.py
index 77842db..8748216 100644
--- a/hyperextract/core/base.py
+++ b/hyperextract/core/base.py
@@ -72,15 +72,15 @@ class BaseAutoType(ABC, Generic[T]):
separators=["\n\n", "\n", "。", "!", "?", ". ", "! ", "? ", " ", ""],
)
+ # Initialize internal state (calls hook for subclass setup)
+ self._init_internal_state()
+
# Internal state storing the extracted knowledge
self.metadata: Dict[str, Any] = {
"created_at": datetime.now(),
"updated_at": datetime.now(),
}
- # Initialize internal state (calls hook for subclass setup)
- self._init_internal_state()
-
def _create_empty_instance(self) -> "BaseAutoType[T]":
"""Creates a new empty instance with the same configuration as this one.
@@ -351,116 +351,34 @@ class BaseAutoType(ABC, Generic[T]):
"""
pass
- # ==================== Serialization: Core Interface ====================
-
- def dump(self, folder_path: str | Path) -> None:
- """Exports knowledge to a specified folder.
-
- Storage Structure:
- folder/
- ├── config.json # Metadata, Schema Info, Parameters
- ├── data.json # Pure Knowledge Data
- └── index/ # Vector Index Files
+ # ==================== Serialization: Public Interface ====================
+ def dump_data(self, file_path: str | Path) -> None:
+ """Saves the pure knowledge data to disk as JSON.
+
Args:
- folder_path: Target folder path for saving.
+ file_path: Target file path for saving data (e.g., "data.json").
"""
- folder = Path(folder_path)
- if folder.is_file():
- raise FileExistsError(f"Path is a file, expected directory: {folder}")
- folder.mkdir(parents=True, exist_ok=True)
-
- # 1. Save Configuration (Schema, Metadata, Parameters)
- self._save_config(folder)
-
- # 2. Save Data (Pure Data)
- self._save_data(folder)
-
- # 3. Save Index (Optional)
- if self._index is not None:
- self._dump_index_storage(folder)
-
- def load(self, folder_path: str | Path) -> None:
- """Loads knowledge from a specified folder.
-
- Args:
- folder_path: Source folder path containing saved knowledge.
- """
- folder = Path(folder_path)
- if not folder.is_dir():
- raise FileNotFoundError(f"Folder not found: {folder_path}")
-
- # 1. Load Configuration (Validate compatibility)
- self._load_config(folder)
-
- # 2. Load Data (Restore state)
- self._load_data(folder)
-
- # 3. Load Index (Restore vector search)
- self._load_index_storage(folder)
-
- # ==================== Serialization: Components ====================
-
- def _save_config(self, folder: Path) -> None:
- """Saves metadata, schema info, and instance parameters."""
- config = {
- "version": "1.0",
- "type": self.__class__.__name__,
- "schema": self._data_schema.__name__,
- "params": {
- "prompt": self.prompt,
- "chunk_size": self.chunk_size,
- "chunk_overlap": self.chunk_overlap,
- "max_workers": self.max_workers,
- },
- "metadata": {
- k: str(v) if isinstance(v, datetime) else v
- for k, v in self.metadata.items()
- },
- "extra": self._get_extra_config_data(),
- }
-
- with open(folder / "config.json", "w", encoding="utf-8") as f:
- json.dump(config, f, indent=2, ensure_ascii=False)
-
- def _save_data(self, folder: Path) -> None:
- """Saves the pure knowledge data."""
+ path = Path(file_path)
+ # Ensure parent directory exists
+ path.parent.mkdir(parents=True, exist_ok=True)
+
export_data = self.data.model_dump()
- with open(folder / "data.json", "w", encoding="utf-8") as f:
+ with open(path, "w", encoding="utf-8") as f:
json.dump(export_data, f, indent=2, ensure_ascii=False, default=str)
- def _load_config(self, folder: Path) -> None:
- """Loads and validates configuration."""
- config_path = folder / "config.json"
- if not config_path.exists():
- raise FileNotFoundError(f"Config file not found: {config_path}")
-
- with open(config_path, "r", encoding="utf-8") as f:
- config = json.load(f)
-
- # Strict schema validation - raise error if mismatch
- if config.get("schema") != self._data_schema.__name__:
- raise ValueError(
- f"Schema mismatch: Expected '{self._data_schema.__name__}', "
- f"but found '{config.get('schema')}' in config.json"
- )
-
- # Restore metadata
- if "metadata" in config:
- self.metadata.update(config["metadata"])
-
- # Restore extra config
- if "extra" in config:
- self._load_extra_config_data(config["extra"])
-
- def _load_data(self, folder: Path) -> None:
- """Loads data and restores internal state."""
- data_path = folder / "data.json"
- if not data_path.exists():
- raise FileNotFoundError(f"Data file not found: {data_path}")
+ def load_data(self, file_path: str | Path) -> None:
+ """Loads data from disk and restores internal state.
+
+ Args:
+ file_path: Source file path containing data (e.g., "data.json").
+ """
+ path = Path(file_path)
+ if not path.exists():
+ raise FileNotFoundError(f"Data file not found: {path}")
- with open(data_path, "r", encoding="utf-8") as f:
+ with open(path, "r", encoding="utf-8") as f:
raw_data = json.load(f)
# Validate and Set State
@@ -468,48 +386,26 @@ class BaseAutoType(ABC, Generic[T]):
self._set_data_state(validated_data)
@abstractmethod
- def _dump_index_storage(self, folder: Path) -> None:
- """HOOK: Save vector index to disk.
+ def dump_index(self, folder_path: str | Path) -> None:
+ """Save vector index to disk.
Subclasses must implement to support their specific vector store
(FAISS, Chroma, Pinecone, etc.).
Args:
- folder: Target folder for saving index files.
+ folder_path: Target folder path for saving index files.
"""
pass
@abstractmethod
- def _load_index_storage(self, folder: Path) -> None:
- """HOOK: Load vector index from disk.
+ def load_index(self, folder_path: str | Path) -> None:
+ """Load vector index from disk.
Subclasses must implement to support their specific vector store
(FAISS, Chroma, Pinecone, etc.).
Args:
- folder: Source folder containing index files.
- """
- pass
-
- # ==================== Serialization: Hooks ====================
-
- def _get_extra_config_data(self) -> Dict[str, Any]:
- """Hook to save subclass-specific configuration (not data).
-
- Override this method to save subclass-specific config like merge strategy.
-
- Returns:
- Dictionary of extra config data.
- """
- return {}
-
- def _load_extra_config_data(self, extra_config: Dict[str, Any]) -> None:
- """Hook to load subclass-specific configuration.
-
- Override this method to restore subclass-specific config.
-
- Args:
- extra_config: Extra configuration data from config.json.
+ folder_path: Source folder path containing index files.
"""
pass
diff --git a/hyperextract/core/list.py b/hyperextract/core/list.py
index a3df171..8ec45e2 100644
--- a/hyperextract/core/list.py
+++ b/hyperextract/core/list.py
@@ -2,32 +2,26 @@
from pathlib import Path
from typing import (
- List,
Any,
+ List,
Type,
+ Union,
TypeVar,
Generic,
- TYPE_CHECKING,
Iterator,
Callable,
Iterable,
- Union,
+ TYPE_CHECKING,
)
from datetime import datetime
from pydantic import BaseModel, Field, create_model
-from langchain_core.language_models.chat_models import BaseChatModel
-from langchain_core.embeddings import Embeddings
from langchain_core.documents import Document
+from langchain_core.embeddings import Embeddings
+from langchain_core.language_models.chat_models import BaseChatModel
from langchain_community.vectorstores import FAISS
from .base import BaseAutoType
-
-try:
- from hyperextract.config import logger
-except ImportError:
- import logging
-
- logger = logging.getLogger(__name__)
+from hyperextract.utils.logging import logger
Item = TypeVar("Item", bound=BaseModel)
@@ -98,6 +92,14 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
),
)
+ # check fields_for_index validity
+ if self.fields_for_index:
+ for field_name in self.fields_for_index:
+ if field_name not in item_schema.model_fields:
+ raise ValueError(
+ f"Field '{field_name}' not found in item schema '{item_schema.__name__}'"
+ )
+
super().__init__(
data_schema=self.item_list_schema,
llm_client=llm_client,
@@ -210,7 +212,7 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
def build_index(self) -> None:
"""Builds independent vector index for each item in the list.
-
+
If fields_for_index is specified, only those fields are indexed.
Otherwise, all fields are indexed.
"""
@@ -228,12 +230,14 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
if self.fields_for_index:
# Only index specified fields
item_dict = item.model_dump()
- indexed_fields = {k: v for k, v in item_dict.items() if k in self.fields_for_index}
+ indexed_fields = {
+ k: v for k, v in item_dict.items() if k in self.fields_for_index
+ }
content = str(indexed_fields)
else:
# Index all fields
content = item.model_dump_json()
-
+
documents.append(
Document(
page_content=content,
@@ -244,7 +248,9 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
if documents:
try:
self._index = FAISS.from_documents(documents, self.embedder)
- logger.info(f"Built FAISS index with {len(documents)} items (fields: {self.fields_for_index or 'all'})")
+ logger.info(
+ f"Built FAISS index with {len(documents)} items (fields: {self.fields_for_index or 'all'})"
+ )
except ImportError:
logger.error("FAISS not available. Install with: pip install faiss-cpu")
raise
@@ -283,17 +289,18 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
# ==================== Index Storage ====================
- def _dump_index_storage(self, folder: Path) -> None:
+ def dump_index(self, folder_path: str | Path) -> None:
"""Saves FAISS vector index to disk."""
if self._index is None:
return
+ folder = Path(folder_path)
+ folder.mkdir(parents=True, exist_ok=True)
index_path = str(folder / "index")
self._index.save_local(index_path)
- def _load_index_storage(self, folder: Path) -> None:
+ def load_index(self, folder_path: str | Path) -> None:
"""Loads FAISS vector index from disk."""
- from langchain_community.vectorstores import FAISS
-
+ folder = Path(folder_path)
index_path = folder / "index"
if index_path.exists():
try:
diff --git a/hyperextract/core/model.py b/hyperextract/core/model.py
index 17fcdf6..88ffde7 100644
--- a/hyperextract/core/model.py
+++ b/hyperextract/core/model.py
@@ -11,7 +11,7 @@ from langchain_community.vectorstores import FAISS
from .base import BaseAutoType, T
try:
- from hyperextract.config import logger
+ from hyperextract.utils.logging import logger
except ImportError:
import logging
@@ -168,18 +168,6 @@ class AutoModel(BaseAutoType[T]):
self._index = FAISS.from_documents(documents, self.embedder)
logger.info(f"Built FAISS index with {len(documents)} documents")
- def __len__(self) -> int:
- """Returns 1 if there is data (non-empty fields), 0 otherwise.
-
- AutoModel always represents a single unit of knowledge, so it's either
- empty (0) or contains one item (1).
- """
- if not self._data:
- return 0
- # Check if any field is non-null
- has_data = any(v is not None for v in self._data.model_dump().values())
- return 1 if has_data else 0
-
def search(self, query: str, top_k: int = 3) -> List[Any]:
"""Searches all indexed fields using semantic similarity.
@@ -214,17 +202,20 @@ class AutoModel(BaseAutoType[T]):
# ==================== Index Storage ====================
- def _dump_index_storage(self, folder: Path) -> None:
+ def dump_index(self, folder_path: str | Path) -> None:
"""Saves FAISS vector index to disk."""
if self._index is None:
return
+ folder = Path(folder_path)
+ folder.mkdir(parents=True, exist_ok=True)
index_path = str(folder / "index")
self._index.save_local(index_path)
- def _load_index_storage(self, folder: Path) -> None:
+ def load_index(self, folder_path: str | Path) -> None:
"""Loads FAISS vector index from disk."""
from langchain_community.vectorstores import FAISS
+ folder = Path(folder_path)
index_path = folder / "index"
if index_path.exists():
try:
diff --git a/hyperextract/core/set.py b/hyperextract/core/set.py
index 686aec8..37ece35 100644
--- a/hyperextract/core/set.py
+++ b/hyperextract/core/set.py
@@ -6,14 +6,14 @@ Supports multiple merge strategies including LLM-powered intelligent merging.
from pathlib import Path
from typing import (
- List,
Any,
+ List,
Type,
+ Union,
TypeVar,
Generic,
Optional,
Callable,
- Union,
TYPE_CHECKING,
)
from datetime import datetime
@@ -26,7 +26,7 @@ from langchain_core.documents import Document
from langchain_community.vectorstores import FAISS
from .base import BaseAutoType
-from hyperextract.config import logger
+from hyperextract.utils.logging import logger
Item = TypeVar("Item", bound=BaseModel)
@@ -48,7 +48,14 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
Key characteristics:
- Extraction target: A unique collection of structured objects
- Deduplication: Based on key_extractor function (user-specified)
- - Merge strategy: Configurable (keep_old/keep_new/field_merge/llm_merge)
+ - Merge strategy: Configurable via MergeStrategy enum:
+ * KEEP_EXISTING: Preserve first (original) data, ignore updates
+ * KEEP_INCOMING: Always use latest data, overwrite existing
+ * MERGE_FIELD: Non-null fields overwrite, lists append (default)
+ * LLM.BALANCED: LLM intelligently synthesizes both versions
+ * LLM.PREFER_EXISTING: LLM synthesis but prioritizes original data
+ * LLM.PREFER_INCOMING: LLM synthesis but prioritizes new data
+ * LLM.CUSTOM_RULE: User-defined rules with dynamic context
- Internal storage: Dict for O(1) lookup and deduplication
- External interface: List (via items property)
- Set operations: union (|), intersection (&), difference (-)
@@ -84,17 +91,16 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
item_schema: Type[Item],
llm_client: BaseChatModel,
embedder: Embeddings,
- *,
key_extractor: Callable[[Item], Any],
+ *,
strategy_or_merger: Union[
MergeStrategy, BaseMerger
- ] = MergeStrategy.LLM.PREFER_INCOMING,
+ ] = MergeStrategy.LLM.BALANCED,
prompt: str = "",
chunk_size: int = 2000,
chunk_overlap: int = 200,
max_workers: int = 10,
show_progress: bool = True,
- llm_batch_size: int = 10,
**kwargs: Any,
):
"""Initialize AutoSet with key extractor and merge strategy.
@@ -112,7 +118,6 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
chunk_overlap: Overlapping characters between adjacent chunks.
max_workers: Maximum concurrent extraction tasks.
show_progress: Whether to log progress information.
- llm_batch_size: Batch size for LLM merge operations (for LLM_MERGE strategy).
**kwargs: Additional arguments passed to create_merger() when strategy_or_merger is
a MergeStrategy enum. Ignored if strategy_or_merger is a BaseMerger instance.
"""
@@ -133,7 +138,6 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
# AutoSet-specific attributes (MUST be initialized BEFORE super().__init__ calls _init_internal_state)
self.key_extractor = key_extractor
self.strategy_or_merger = strategy_or_merger
- self.llm_batch_size = llm_batch_size
# Setup Merge Strategy
if isinstance(strategy_or_merger, BaseMerger):
@@ -155,7 +159,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
)
# Initialize OMem instance BEFORE calling super().__init__ so _init_internal_state can use it
- self._data: OMem[Item] = OMem(
+ self._data_memory: OMem[Item] = OMem(
memory_schema=item_schema,
key_extractor=key_extractor,
llm_client=llm_client,
@@ -196,7 +200,6 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
chunk_overlap=self.chunk_overlap,
max_workers=self.max_workers,
show_progress=self.show_progress,
- llm_batch_size=self.llm_batch_size,
)
def _default_prompt(self) -> str:
@@ -216,7 +219,10 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
Returns:
The internal knowledge data as a Pydantic model instance.
"""
- return self.data_schema(items=self.items)
+ if len(self.data.items) != len(self.items):
+ # Sync data from OMem
+ self._data = self.data_schema(items=self.items)
+ return self._data
@property
def items(self) -> List[Item]:
@@ -225,7 +231,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
Returns:
List of unique items.
"""
- return self._data.items
+ return self._data_memory.items
@property
def keys(self) -> List[Any]:
@@ -234,43 +240,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
Returns:
List of unique key values.
"""
- return self._data.keys
-
- def __len__(self) -> int:
- """Returns the number of unique items in the set."""
- return len(self._data.items)
-
- def __contains__(self, key: Any) -> bool:
- """Checks if a unique key exists in the set.
-
- Args:
- key: The unique key value to check.
-
- Returns:
- True if key exists, False otherwise.
- """
- return self._data.get(key) is not None
-
- def __repr__(self) -> str:
- """Returns a developer-friendly representation."""
- return f"AutoSet[{self.item_schema.__name__}]({len(self)} unique items)"
-
- def __str__(self) -> str:
- """Returns a user-friendly string representation."""
- return f"AutoSet with {len(self)} unique {self.item_schema.__name__} items"
-
- def __iter__(self):
- """Enables iteration over all items in the set.
-
- Returns:
- Iterator over all unique items.
-
- Examples:
- >>> for skill in skills:
- ... print(skill.name)
- >>> names = [s.name for s in skills]
- """
- return iter(self.items)
+ return self._data_memory.keys
# ==================== State Management Lifecycle Overrides ====================
@@ -279,20 +249,21 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
INIT/RESET: Initialize or reset OMem as empty.
Called during __init__ and when clear() is called.
"""
- self._data.clear()
+ self._data = self.item_set_schema()
+ self._data_memory.clear()
def _init_index_state(self) -> None:
"""Initialize vector index to empty state."""
- self._index = None
+ self._data_memory.clear_index()
def _set_data_state(self, data: AutoSetSchema[Item]) -> None:
"""
SET: Full Reset. Wipe OMem and refill from data (e.g., load from disk).
Called by extract() or load() where data IS the new state.
"""
- self._data.clear()
+ self._data_memory.clear()
if data.items:
- self._data.add(data.items)
+ self._data_memory.add(data.items)
self.clear_index()
def _update_data_state(self, incoming_data: AutoSetSchema[Item]) -> None:
@@ -304,7 +275,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
handles deduplication and merging internally.
"""
if incoming_data.items:
- self._data.add(incoming_data.items)
+ self._data_memory.add(incoming_data.items)
self.clear_index()
# ==================== Core Override Methods ====================
@@ -336,57 +307,10 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
)
merged_items = self._merger.merge(all_items)
-
- # Get strategy value for logging
- if isinstance(self.strategy_or_merger, BaseMerger):
- strategy_str = (
- self.strategy_or_merger.strategy
- if hasattr(self.strategy_or_merger, "strategy")
- else "custom_merger"
- )
- else:
- strategy_str = (
- self.strategy_or_merger.value
- if hasattr(self.strategy_or_merger, "value")
- else str(self.strategy_or_merger)
- )
-
- logger.info(
- f"Merged into {len(merged_items)} unique items (strategy: {strategy_str})"
- )
+ logger.info(f"Merged into {len(merged_items)} unique items.")
return self.item_set_schema(items=merged_items)
- # ==================== Serialization Helpers ====================
-
- def _get_extra_dump_data(self) -> dict:
- """Add AutoSet specific data to the dump.
-
- Returns:
- Dictionary with extra AutoSet-specific fields.
- """
- # Get strategy value: handle different types of strategy_or_merger
- if isinstance(self.strategy_or_merger, BaseMerger):
- strategy_value = (
- self.strategy_or_merger.strategy
- if hasattr(self.strategy_or_merger, "strategy")
- else "custom_merger"
- )
- elif isinstance(self.strategy_or_merger, str):
- strategy_value = self.strategy_or_merger
- else:
- # MergeStrategy enum
- strategy_value = (
- self.strategy_or_merger.value
- if hasattr(self.strategy_or_merger, "value")
- else str(self.strategy_or_merger)
- )
-
- return {
- "item_schema_name": self.item_schema.__name__,
- "strategy_or_merger": strategy_value,
- }
-
# ==================== Indexing & Query ====================
def build_index(self) -> None:
@@ -452,17 +376,20 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
# ==================== Index Storage ====================
- def _dump_index_storage(self, folder: Path) -> None:
+ def dump_index(self, folder_path: str | Path) -> None:
"""Saves FAISS vector index to disk."""
if self._index is None:
return
+ folder = Path(folder_path)
+ folder.mkdir(parents=True, exist_ok=True)
index_path = str(folder / "index")
self._index.save_local(index_path)
- def _load_index_storage(self, folder: Path) -> None:
+ def load_index(self, folder_path: str | Path) -> None:
"""Loads FAISS vector index from disk."""
from langchain_community.vectorstores import FAISS
+ folder = Path(folder_path)
index_path = folder / "index"
if index_path.exists():
try:
@@ -474,6 +401,42 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
else:
self._index = None
+ def __len__(self) -> int:
+ """Returns the number of unique items in the set."""
+ return len(self._data_memory.items)
+
+ def __contains__(self, key: Any) -> bool:
+ """Checks if a unique key exists in the set.
+
+ Args:
+ key: The unique key value to check.
+
+ Returns:
+ True if key exists, False otherwise.
+ """
+ return self._data_memory.get(key) is not None
+
+ def __repr__(self) -> str:
+ """Returns a developer-friendly representation."""
+ return f"AutoSet[{self.item_schema.__name__}]({len(self)} unique items)"
+
+ def __str__(self) -> str:
+ """Returns a user-friendly string representation."""
+ return f"AutoSet with {len(self)} unique {self.item_schema.__name__} items"
+
+ def __iter__(self):
+ """Enables iteration over all items in the set.
+
+ Returns:
+ Iterator over all unique items.
+
+ Examples:
+ >>> for skill in skills:
+ ... print(skill.name)
+ >>> names = [s.name for s in skills]
+ """
+ return iter(self.items)
+
# ==================== Set-Specific Methods ====================
def add(self, item: Item) -> None:
@@ -483,7 +446,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
item: The item to add.
"""
# OMem handles deduplication and merging automatically
- self._data.add([item])
+ self._data_memory.add([item])
self.clear_index()
self.metadata["updated_at"] = datetime.now()
@@ -497,12 +460,12 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
The removed item, or None if not found.
"""
# Get item before removing
- item = self._data.get(key)
+ item = self._data_memory.get(key)
if item is None:
return None
# Remove from OMem
- removed = self._data.remove(key)
+ removed = self._data_memory.remove(key)
if removed:
self.clear_index()
self.metadata["updated_at"] = datetime.now()
@@ -518,7 +481,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
Returns:
True if key exists, False otherwise.
"""
- return self._data.get(key) is not None
+ return self._data_memory.get(key) is not None
def get(self, key: Any, default: Optional[Item] = None) -> Optional[Item]:
"""Gets an item by its unique key value.
@@ -530,7 +493,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
Returns:
The item if found, otherwise default.
"""
- result = self._data.get(key)
+ result = self._data_memory.get(key)
return result if result is not None else default
def update(self, items: List[Item]) -> None:
@@ -558,8 +521,8 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
- Updates metadata timestamp
"""
try:
- if self._data.get(key) is not None:
- self._data.remove(key)
+ if self._data_memory.get(key) is not None:
+ self._data_memory.remove(key)
self.clear_index()
self.metadata["updated_at"] = datetime.now()
except (KeyError, Exception):
@@ -585,12 +548,12 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
- Clears the vector index (needs rebuild)
- Updates metadata timestamp
"""
- if not self._data.items:
+ if not self._data_memory.items:
raise KeyError("pop from an empty AutoSet")
- item = self._data.items[0]
+ item = self._data_memory.items[0]
key = self.key_extractor(item)
- self._data.remove(key)
+ self._data_memory.remove(key)
self.clear_index()
self.metadata["updated_at"] = datetime.now()
@@ -611,8 +574,8 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
new_set = self._create_empty_instance()
# Add copies of all items
- items_copy = [item.model_copy(deep=True) for item in self._data.items]
- new_set._data.add(items_copy)
+ items_copy = [item.model_copy(deep=True) for item in self._data_memory.items]
+ new_set._data_memory.add(items_copy)
new_set.metadata = self.metadata.copy()
new_set.metadata["created_at"] = datetime.now()
@@ -646,8 +609,8 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
new_set = self._create_empty_instance()
# Add all items from both sets (OMem handles merge automatically)
- all_items = self._data.items + other._data.items
- new_set._data.add(all_items)
+ all_items = self._data_memory.items + other._data_memory.items
+ new_set._data_memory.add(all_items)
new_set.metadata["updated_at"] = datetime.now()
return new_set
@@ -679,11 +642,13 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
# Only keep items present in both sets (by key)
intersection_items = [
- item for item in self._data.items if self.key_extractor(item) in other.keys
+ item
+ for item in self._data_memory.items
+ if self.key_extractor(item) in other.keys
]
if intersection_items:
- new_set._data.add(intersection_items)
+ new_set._data_memory.add(intersection_items)
new_set.metadata["updated_at"] = datetime.now()
@@ -717,12 +682,12 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
# Only keep items not in other (by key)
difference_items = [
item
- for item in self._data.items
+ for item in self._data_memory.items
if self.key_extractor(item) not in other.keys
]
if difference_items:
- new_set._data.add(difference_items)
+ new_set._data_memory.add(difference_items)
new_set.metadata["updated_at"] = datetime.now()
@@ -756,7 +721,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
# Items only in self
symmetric_items = [
item
- for item in self._data.items
+ for item in self._data_memory.items
if self.key_extractor(item) not in other.keys
]
@@ -764,13 +729,13 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
symmetric_items.extend(
[
item
- for item in other._data.items
+ for item in other._data_memory.items
if self.key_extractor(item) not in self.keys
]
)
if symmetric_items:
- new_set._data.add(symmetric_items)
+ new_set._data_memory.add(symmetric_items)
new_set.metadata["updated_at"] = datetime.now()
diff --git a/hyperextract/utils/__init__.py b/hyperextract/utils/__init__.py
index 596ffa7..8e8a47b 100644
--- a/hyperextract/utils/__init__.py
+++ b/hyperextract/utils/__init__.py
@@ -8,6 +8,7 @@ from .merger import (
FieldMerger,
LLMMerger,
)
+from .logging import logger, setup_logger
__all__ = [
"BaseMerger",
@@ -16,4 +17,6 @@ __all__ = [
"KeepOldMerger",
"FieldMerger",
"LLMMerger",
+ "logger",
+ "setup_logger",
]
diff --git a/hyperextract/utils/logging.py b/hyperextract/utils/logging.py
new file mode 100644
index 0000000..d5c8e09
--- /dev/null
+++ b/hyperextract/utils/logging.py
@@ -0,0 +1,101 @@
+"""
+Loguru Logger Configuration
+
+Provides flexible logging setup with optional file output.
+Supports environment variables for configuration without requiring code changes.
+"""
+
+import os
+import sys
+from pathlib import Path
+from typing import Optional, Union
+from loguru import logger
+
+# ==================== Environment Variable Constants ====================
+ENV_LOG_LEVEL = "HYPER_EXTRACT_LOG_LEVEL"
+ENV_LOG_FILE = "HYPER_EXTRACT_LOG_FILE"
+
+
+def setup_logger(
+ level: str = "INFO",
+ log_file: Optional[Union[str, Path]] = None,
+ rotation: str = "10 MB",
+ retention: str = "10 days",
+ serialize: bool = False,
+) -> None:
+ """
+ Configure Loguru Logger with optional file output.
+
+ This function can be called explicitly by users to override default configuration.
+ Environment variables are checked first, then parameters, then defaults.
+
+ Args:
+ level: Log level (e.g., "DEBUG", "INFO", "WARNING", "ERROR").
+ Defaults to "INFO". Overridden by HYPER_EXTRACT_LOG_LEVEL env var.
+ log_file: Path to log file. If None, only console output is used.
+ Defaults to None. Overridden by HYPER_EXTRACT_LOG_FILE env var.
+ rotation: Log rotation size (e.g., "10 MB", "1 GB", "00:00").
+ retention: Log retention duration (e.g., "10 days", "7 days").
+ serialize: If True, outputs logs in JSON format (suitable for production).
+
+ Examples:
+ # Default: console only, INFO level
+ setup_logger()
+
+ # Add file logging via code
+ setup_logger(level="DEBUG", log_file="logs/debug.log")
+
+ # Using environment variables (recommended for deployment)
+ # HYPER_EXTRACT_LOG_LEVEL=DEBUG
+ # HYPER_EXTRACT_LOG_FILE=./logs/app.log
+ """
+ # Remove all existing handlers to prevent duplication
+ logger.remove()
+
+ # 1. Determine final configuration (environment variables have highest priority)
+ final_level = os.getenv(ENV_LOG_LEVEL, level).upper()
+ env_log_file = os.getenv(ENV_LOG_FILE)
+ final_log_file = log_file if log_file else env_log_file
+
+ # 2. Add console handler
+ console_format = (
+ "<green>{time:YYYY-MM-DD HH:mm:ss}</green> | "
+ "<level>{level: <8}</level> | "
+ "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - "
+ "<level>{message}</level>"
+ )
+
+ logger.add(
+ sys.stderr,
+ level=final_level,
+ format=console_format,
+ serialize=serialize,
+ colorize=True,
+ )
+
+ # 3. Add file handler (only if path is configured)
+ if final_log_file:
+ log_path = Path(final_log_file)
+ # Ensure parent directory exists
+ log_path.parent.mkdir(parents=True, exist_ok=True)
+
+ logger.add(
+ str(log_path),
+ rotation=rotation,
+ retention=retention,
+ level="DEBUG", # File captures more detailed information
+ compression="zip",
+ encoding="utf-8",
+ enqueue=True, # Asynchronous I/O for thread safety
+ serialize=serialize,
+ )
+
+
+# ==================== Default Initialization ====================
+# Initialize with sensible defaults on module import.
+# This ensures users don't need explicit setup_logger() calls for basic usage.
+# Environment variables will be checked and applied automatically.
+setup_logger(level="INFO")
+
+# Export public API
+__all__ = ["logger", "setup_logger"]
diff --git a/hyperextract/utils/merger/base.py b/hyperextract/utils/merger/base.py
index d53d778..aee28fd 100644
--- a/hyperextract/utils/merger/base.py
+++ b/hyperextract/utils/merger/base.py
@@ -12,7 +12,7 @@ from typing import List, Any, Callable, TypeVar, Generic, Dict, Tuple, Optional
from collections import defaultdict
try:
- from hyperextract.config import logger
+ from hyperextract.utils.logging import logger
except ImportError:
logger = logging.getLogger(__name__)
diff --git a/hyperextract/utils/merger/llm.py b/hyperextract/utils/merger/llm.py
index f92e51d..2f3932f 100644
--- a/hyperextract/utils/merger/llm.py
+++ b/hyperextract/utils/merger/llm.py
@@ -17,7 +17,7 @@ from langchain_core.prompts import ChatPromptTemplate
from .base import BaseMerger
try:
- from hyperextract.config import logger
+ from hyperextract.utils.logging import logger
except ImportError:
logger = logging.getLogger(__name__)
diff --git a/hyperextract/utils/merger/standard.py b/hyperextract/utils/merger/standard.py
index 494c83b..feeeb05 100644
--- a/hyperextract/utils/merger/standard.py
+++ b/hyperextract/utils/merger/standard.py
@@ -11,7 +11,7 @@ from typing import List, TypeVar
from .base import BaseMerger
try:
- from hyperextract.config import logger
+ from hyperextract.utils.logging import logger
except ImportError:
logger = logging.getLogger(__name__)