Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
重构状态管理方法,统一命名为 _init_data_state、_set_data_state 和 _update_data_state,增强代码一致性和可读性
yifanfeng9774d69bd4a8c0
diff --git a/examples/auto_set_demo.py b/examples/auto_set_demo.pyindex 0f73b66..5ee460d 100644--- a/examples/auto_set_demo.py+++ b/examples/auto_set_demo.py@@ -66,7 +66,7 @@ def main():llm_client=llm,embedder=embedder,key_extractor=lambda x: x.name, # 基于怪物名称去重- merge_item_strategy=MergeStrategy.LLM_MERGE,+ strategy_or_merger="llm_prefer_incoming",prompt="""从冒险者的日志记录中提取所有怪物的相关信息。对于每只怪物,请提供:diff --git a/hyperextract/core/base.py b/hyperextract/core/base.pyindex 85fbcc8..e6aaf60 100644--- a/hyperextract/core/base.py+++ b/hyperextract/core/base.py@@ -30,6 +30,8 @@ class BaseAutoType(ABC, Generic[T]):- Provide serialization and deserialization capabilities"""+ # ==================== Initialization & Configuration ====================+def __init__(self,data_schema: Type[T],@@ -77,11 +79,7 @@ class BaseAutoType(ABC, Generic[T]):separators=["\n\n", "\n", "。", "!", "?", ". ", "! ", "? ", " ", ""],)- # Vector index using FAISS for semantic search- self._index = None-# Internal state storing the extracted knowledge- # NOTE: self._data will be initialized by _init_internal_state hook (subclass responsibility)self.metadata: Dict[str, Any] = {"created_at": datetime.now(),"updated_at": datetime.now(),@@ -112,12 +110,12 @@ class BaseAutoType(ABC, Generic[T]):@abstractmethoddef _default_prompt(self) -> str:"""Returns the default extraction prompt template.-+Subclasses must implement this to provide a prompt tailored to their extraction pattern."""pass- # ==================== Data Management ====================+ # ==================== Data Access Interface ====================@propertydef data_schema(self) -> Type[T]:@@ -142,30 +140,53 @@ class BaseAutoType(ABC, Generic[T]):"""pass- # ==================== State Management Lifecycle Hooks ====================+ # ==================== Data Management Operations ====================++ def clear(self):+ """Clears all knowledge including data and vector index."""+ self._init_internal_state()+ self.metadata["updated_at"] = datetime.now()++ def clear_index(self):+ """Clears the vector index without affecting the stored data."""+ self._init_index_state()++ # ==================== Lifecycle Hooks: State Management ====================- @abstractmethoddef _init_internal_state(self) -> None:+ """Master control: Initialize or reset all internal state (INIT/RESET).++ This concrete method orchestrates the reset process by calling two hooks:+ 1. _init_data_state() - Subclass responsibility: reset data structures+ 2. _init_index_state() - Default: reset index, can be overridden by subclass++ Called in two scenarios:+ - During __init__ to set up the initial state+ - When clear() is called to reset to empty state"""- Protected hook to initialize internal state (INIT).- Called at the END of __init__ to ensure all basic attributes are set first.+ self._init_data_state()+ self._init_index_state()- Responsibilities:- 1. Initialize self._data with appropriate structure (may differ from Schema T)- 2. Initialize self._index = None+ @abstractmethod+ def _init_data_state(self) -> None:+ """HOOK: Initialize or reset data structures to empty state.++ Subclass Responsibility:+ - Initialize self._data with appropriate structure (may differ from Schema T)+ - Reset any auxiliary data structures (e.g., lookup dicts, caches)Subclasses must implement this to set up internal structures that may be optimized- beyond the standard Pydantic schema (e.g., dict-based for AutoSet, graph structures).+ beyond the standard Pydantic schema (e.g., OMem for AutoSet, dict-based for others)."""pass@abstractmethod- def _set_internal_state(self, data: T) -> None:- """- Protected hook to overwrite internal state (SET).+ def _set_data_state(self, data: T) -> None:+ """HOOK: Overwrite data state with new data (SET).+Called by extract() or load() where the data provided IS the new state.- Responsibilities:+ Subclass Responsibilities:1. Replace self._data with new data (full reset)2. Convert standard Schema T to optimized internal structure if needed3. Invalidate vector index (self.clear_index())@@ -176,13 +197,13 @@ class BaseAutoType(ABC, Generic[T]):pass@abstractmethod- def _update_internal_state(self, incoming_data: T) -> None:- """- Protected hook to merge new data into state (UPDATE).+ def _update_data_state(self, incoming_data: T) -> None:+ """HOOK: Merge new data into state (UPDATE).+Called by feed() where the data provided is INCREMENTAL.- Responsibilities:- 1. Merge incoming_data into current internal state (optimized for incremental updates)+ Subclass Responsibilities:+ 1. Merge incoming_data into current data state (optimized for incremental updates)2. Invalidate vector index (self.clear_index())Subclasses should implement optimized incremental updates (e.g., set.add instead of@@ -194,27 +215,18 @@ class BaseAutoType(ABC, Generic[T]):pass@abstractmethod- def _clear_internal_state(self) -> None:- """- Protected hook to fully clear internal state (CLEAR).- Called by clear() method.+ def _init_index_state(self) -> None:+ """HOOK: Initialize or reset vector index to empty state.- Responsibilities:- 1. Reset internal state to empty (matching _init_internal_state)+ Subclass Responsibility:+ - Initialize or reset vector index structures+ - Can be FAISS, Chroma, Pinecone, or custom implementation+ - Typically sets self._index = None or initializes specific index instance++ This separation allows index implementation to be decoupled from base class."""pass- # ==================== Data Management ====================-- def clear(self):- """Clears all knowledge including data and vector index."""- self._clear_internal_state()- self.metadata["updated_at"] = datetime.now()-- def clear_index(self):- """Clears the vector index without affecting the stored data."""- self._index = None-# ==================== Extraction & Merge ====================def _extract_data(self, text: str) -> T:@@ -256,7 +268,7 @@ class BaseAutoType(ABC, Generic[T]):extracted_data = self._extract_data(text)new_instance = self._create_empty_instance()- new_instance._set_internal_state(extracted_data)+ new_instance._set_data_state(extracted_data)new_instance.metadata["created_at"] = datetime.now()new_instance.metadata["updated_at"] = datetime.now()@@ -277,10 +289,10 @@ class BaseAutoType(ABC, Generic[T]):Self (the current instance)."""extracted_data = self._extract_data(text)-+# Use UPDATE hook instead of manual merge+set- self._update_internal_state(extracted_data)-+ self._update_data_state(extracted_data)+self.metadata["updated_at"] = datetime.now()return self@@ -307,7 +319,7 @@ class BaseAutoType(ABC, Generic[T]):"""pass- # ==================== Indexing & Query ====================+ # ==================== Indexing & Search ====================@abstractmethoddef build_index(self):@@ -338,14 +350,14 @@ class BaseAutoType(ABC, Generic[T]):"""pass- # ==================== Serialization ====================+ # ==================== Serialization: Core Interface ====================def dump(self, folder_path: str | Path) -> None:"""Exports knowledge to a specified folder.Saves to the folder:1. Structured data (self._data) - saved as state.json- 2. Vector index (self._index) - saved as FAISS index files+ 2. Vector index (self._index) - saved via _dump_index_storage hookArgs:folder_path: Target folder path for saving.@@ -374,10 +386,9 @@ class BaseAutoType(ABC, Generic[T]):with open(data_file, "w", encoding="utf-8") as f:f.write(json_str)- # 2. Save vector index+ # 2. Save vector index (via hook for flexibility)if self._index is not None:- index_path = str(folder / "faiss_index")- self._index.save_local(index_path)+ self._dump_index_storage(folder)except Exception:raise@@ -387,13 +398,11 @@ class BaseAutoType(ABC, Generic[T]):Loads from the folder:1. Structured data (self._data) - loaded from state.json- 2. Vector index (self._index) - loaded from FAISS index files+ 2. Vector index (self._index) - loaded via _load_index_storage hookArgs:folder_path: Source folder path containing saved knowledge."""- from langchain_community.vectorstores import FAISS-folder = Path(folder_path)if not folder.is_dir():raise ValueError(f"Folder does not exist: {folder_path}")@@ -408,7 +417,7 @@ class BaseAutoType(ABC, Generic[T]):# Load data using hook for proper state syncloaded_data = self._data_schema.model_validate(data.get("data", {}))- self._set_internal_state(loaded_data)+ self._set_data_state(loaded_data)# Update metadata with loaded valuesif "metadata" in data:@@ -418,7 +427,42 @@ class BaseAutoType(ABC, Generic[T]):# Allow subclasses to load extra dataself._load_extra_data(data)- # 2. Load vector index+ # 2. Load vector index (via hook for flexibility)+ self._load_index_storage(folder)++ # ==================== Serialization: Index Storage Hooks ====================++ def _dump_index_storage(self, folder: Path) -> None:+ """HOOK: Save vector index to disk.++ Default Implementation: FAISS save_local++ Subclasses can override to support:+ - Alternative vector store implementations (Chroma, Pinecone, etc.)+ - Custom serialization logic+ - Multiple indices++ Args:+ folder: Target folder for saving index files.+ """+ index_path = str(folder / "faiss_index")+ self._index.save_local(index_path)++ def _load_index_storage(self, folder: Path) -> None:+ """HOOK: Load vector index from disk.++ Default Implementation: FAISS load_local++ Subclasses can override to support:+ - Alternative vector store implementations (Chroma, Pinecone, etc.)+ - Custom deserialization logic+ - Multiple indices++ Args:+ folder: Source folder containing index files.+ """+ from langchain_community.vectorstores import FAISS+index_path = str(folder / "faiss_index")if Path(index_path).exists():try:@@ -427,8 +471,10 @@ class BaseAutoType(ABC, Generic[T]):)except Exception:self._index = None+ else:+ self._index = None- # ==================== Serialization Helpers ====================+ # ==================== Serialization: Helper Methods ====================def _prepare_metadata_for_dump(self) -> Dict[str, Any]:"""Helper to serialize metadata values (e.g., datetimes)."""@@ -505,7 +551,7 @@ class BaseAutoType(ABC, Generic[T]):new_instance = self._create_empty_instance()# Set the merged data using hook and update metadata- new_instance._set_internal_state(merged_data)+ new_instance._set_data_state(merged_data)new_instance.metadata["created_at"] = min(self.metadata["created_at"], other.metadata["created_at"])diff --git a/hyperextract/core/list.py b/hyperextract/core/list.pyindex 2810d1a..6c88478 100644--- a/hyperextract/core/list.py+++ b/hyperextract/core/list.py@@ -151,14 +151,18 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):# ==================== State Management Lifecycle Hooks ====================- def _init_internal_state(self) -> None:+ def _init_data_state(self) -> None:"""- INIT: Initialize with empty schema.+ INIT/RESET: Initialize or reset with empty schema.+ Called during __init__ and when clear() is called."""self._data = self.item_list_schema()++ def _init_index_state(self) -> None:+ """Initialize vector index to empty state."""self._index = None- def _set_internal_state(self, data: AutoListSchema) -> None:+ def _set_data_state(self, data: AutoListSchema) -> None:"""SET: Full reset. Replace with new data (e.g., load from disk).Called by extract() or load() where data IS the new state.@@ -166,7 +170,7 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):self._data = dataself.clear_index()- def _update_internal_state(self, incoming_data: AutoListSchema) -> None:+ def _update_data_state(self, incoming_data: AutoListSchema) -> None:"""UPDATE: Incremental merge. Append incoming items to current list (called by feed()).@@ -177,13 +181,6 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):self._data = merged_dataself.clear_index()- def _clear_internal_state(self) -> None:- """- CLEAR: Reset to empty schema instance.- """- self._data = self.item_list_schema()- self.clear_index()-# ==================== Core Methods ====================def merge_batch(self, data_list: List[AutoListSchema]) -> AutoListSchema:diff --git a/hyperextract/core/model.py b/hyperextract/core/model.pyindex 1549a37..eac53b9 100644--- a/hyperextract/core/model.py+++ b/hyperextract/core/model.py@@ -90,14 +90,18 @@ class AutoModel(BaseAutoType[T]):# ==================== State Management Lifecycle Hooks ====================- def _init_internal_state(self) -> None:+ def _init_data_state(self) -> None:"""- INIT: Initialize with empty schema instance.+ INIT/RESET: Initialize or reset with empty schema instance.+ Called during __init__ and when clear() is called."""self._data = self._data_schema()++ def _init_index_state(self) -> None:+ """Initialize vector index to empty state."""self._index = None- def _set_internal_state(self, data: T) -> None:+ def _set_data_state(self, data: T) -> None:"""SET: Full reset. Replace with new data (e.g., load from disk).Called by extract() or load() where data IS the new state.@@ -105,7 +109,7 @@ class AutoModel(BaseAutoType[T]):self._data = dataself.clear_index()- def _update_internal_state(self, incoming_data: T) -> None:+ def _update_data_state(self, incoming_data: T) -> None:"""UPDATE: Incremental merge. Merge fields with field-level update strategy (called by feed()).@@ -115,13 +119,6 @@ class AutoModel(BaseAutoType[T]):self._data = merged_dataself.clear_index()- def _clear_internal_state(self) -> None:- """- CLEAR: Reset to empty schema instance.- """- self._data = self._data_schema()- self.clear_index()-# ==================== Core Methods ====================def merge_batch(self, data_list: List[T]) -> T:diff --git a/hyperextract/core/set.py b/hyperextract/core/set.pyindex 3ae957c..9038bf4 100644--- a/hyperextract/core/set.py+++ b/hyperextract/core/set.py@@ -4,13 +4,15 @@ Provides automatic deduplication based on a user-specified unique key field.Supports multiple merge strategies including LLM-powered intelligent merging."""-from typing import List, Any, Type, TypeVar, Generic, Optional, Callable+from typing import List, Any, Type, TypeVar, Generic, Optional, Callable, Unionfrom datetime import datetimefrom ontomem import OMem-from ontomem.merger import MergeStrategy, create_merger+from ontomem.merger import MergeStrategy, create_merger, BaseMergerfrom pydantic import BaseModel, Field, create_modelfrom langchain_core.language_models.chat_models import BaseChatModelfrom langchain_core.embeddings import Embeddings+from langchain_core.documents import Document+from langchain_community.vectorstores import FAISSfrom .base import BaseAutoType@@ -75,13 +77,14 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):embedder: Embeddings,*,key_extractor: Callable[[Item], Any],- merge_item_strategy: MergeStrategy = MergeStrategy.LLM.PREFER_INCOMING,+ strategy_or_merger: Union[MergeStrategy, BaseMerger] = MergeStrategy.LLM.PREFER_INCOMING,prompt: str = "",chunk_size: int = 2000,chunk_overlap: int = 200,max_workers: int = 10,show_progress: bool = True,llm_batch_size: int = 10,+ **kwargs: Any,):"""Initialize AutoSet with key extractor and merge strategy.@@ -90,13 +93,17 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):llm_client: Language model client for extraction and merging.embedder: Embedding model for vector indexing.key_extractor: Function to extract unique key from an item (required).- merge_item_strategy: Strategy for merging duplicate items (default: KEEP_NEW).+ strategy_or_merger: Merge strategy or pre-configured merger instance. Can be:+ 1. A MergeStrategy enum value (e.g., MergeStrategy.LLM.BALANCED)+ 2. A pre-configured BaseMerger instance (for full control)prompt: Custom extraction prompt.chunk_size: Maximum characters per chunk for long texts.chunk_overlap: Overlapping characters between adjacent chunks.max_workers: Maximum concurrent extraction tasks.show_progress: Whether to log progress information.llm_batch_size: Batch size for LLM merge operations (for LLM_MERGE strategy).+ **kwargs: Additional arguments passed to create_merger() when strategy_or_merger is+ a MergeStrategy enum. Ignored if strategy_or_merger is a BaseMerger instance."""# Store item_schema@@ -112,31 +119,31 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):),)- super().__init__(- data_schema=self.item_set_schema,- llm_client=llm_client,- embedder=embedder,- prompt=prompt or self._default_prompt(),- chunk_size=chunk_size,- chunk_overlap=chunk_overlap,- max_workers=max_workers,- show_progress=show_progress,- )-- # AutoSet-specific attributes (before _init_internal_state is called by parent)+ # AutoSet-specific attributes (MUST be initialized BEFORE super().__init__ calls _init_internal_state)self.key_extractor = key_extractor- self.merge_item_strategy = merge_item_strategy+ self.strategy_or_merger = strategy_or_mergerself.llm_batch_size = llm_batch_size- # Create Merger instance using ontomem's create_merger- self._merger = create_merger(- strategy=merge_item_strategy,- key_extractor=key_extractor,- llm_client=llm_client,- item_schema=item_schema,- )+ # Setup Merge Strategy+ if isinstance(strategy_or_merger, BaseMerger):+ # Pre-configured merger instance: use directly+ if kwargs:+ logger.warning(+ "Initialized with a Merger instance. Additional kwargs are ignored: %s",+ list(kwargs.keys())+ )+ self._merger = strategy_or_merger+ else:+ # MergeStrategy enum: create merger with strategy and pass kwargs+ self._merger = create_merger(+ strategy=strategy_or_merger,+ key_extractor=key_extractor,+ llm_client=llm_client,+ item_schema=item_schema,+ **kwargs, # Pass additional arguments to create_merger+ )- # Initialize OMem instance for storage and deduplication+ # Initialize OMem instance BEFORE calling super().__init__ so _init_internal_state can use itself._data: OMem[Item] = OMem(memory_schema=item_schema,key_extractor=key_extractor,@@ -146,6 +153,17 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):verbose=show_progress,)+ super().__init__(+ data_schema=self.item_set_schema,+ llm_client=llm_client,+ embedder=embedder,+ prompt=prompt or self._default_prompt(),+ chunk_size=chunk_size,+ chunk_overlap=chunk_overlap,+ max_workers=max_workers,+ show_progress=show_progress,+ )+# ==================== Override Instance Creation ====================def _create_empty_instance(self) -> "AutoSet[Item]":@@ -161,7 +179,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):llm_client=self.llm_client,embedder=self.embedder,key_extractor=self.key_extractor,- merge_item_strategy=self.merge_item_strategy,+ strategy_or_merger=self.strategy_or_merger,prompt=self.prompt,chunk_size=self.chunk_size,chunk_overlap=self.chunk_overlap,@@ -209,7 +227,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):def __len__(self) -> int:"""Returns the number of unique items in the set."""- return len(self._data)+ return len(self._data.items)def __contains__(self, key: Any) -> bool:"""Checks if a unique key exists in the set.@@ -245,14 +263,18 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):# ==================== State Management Lifecycle Overrides ====================- def _init_internal_state(self) -> None:+ def _init_data_state(self) -> None:"""- INIT: Initialize OMem as empty.+ INIT/RESET: Initialize or reset OMem as empty.+ Called during __init__ and when clear() is called."""self._data.clear()- self.clear_index()- def _set_internal_state(self, data: AutoSetSchema[Item]) -> None:+ def _init_index_state(self) -> None:+ """Initialize vector index to empty state."""+ self._index = None++ def _set_data_state(self, data: AutoSetSchema[Item]) -> None:"""SET: Full Reset. Wipe OMem and refill from data (e.g., load from disk).Called by extract() or load() where data IS the new state.@@ -262,7 +284,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):self._data.add(data.items)self.clear_index()- def _update_internal_state(self, incoming_data: AutoSetSchema[Item]) -> None:+ def _update_data_state(self, incoming_data: AutoSetSchema[Item]) -> None:"""UPDATE: Incremental merge. Add to OMem efficiently (called by feed()).@@ -274,13 +296,6 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):self._data.add(incoming_data.items)self.clear_index()- def _clear_internal_state(self) -> None:- """- CLEAR: Wipe OMem (called by clear()).- """- self._data.clear()- self.clear_index()-# ==================== Core Override Methods ====================def merge_batch(self, data_list: List[AutoSetSchema[Item]]) -> AutoSetSchema[Item]:@@ -309,9 +324,15 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):merged_items = self._merger.merge(all_items)+ # Get strategy value for logging+ if isinstance(self.strategy_or_merger, BaseMerger):+ strategy_str = self.strategy_or_merger.strategy if hasattr(self.strategy_or_merger, 'strategy') else "custom_merger"+ else:+ strategy_str = self.strategy_or_merger.value if hasattr(self.strategy_or_merger, 'value') else str(self.strategy_or_merger)+logger.info(f"Merged into {len(merged_items)} unique items "- f"(strategy: {self.merge_item_strategy.value})"+ f"(strategy: {strategy_str})")return self.item_set_schema(items=merged_items)@@ -324,11 +345,83 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):Returns:Dictionary with extra AutoSet-specific fields."""+ # Get strategy value: handle different types of strategy_or_merger+ if isinstance(self.strategy_or_merger, BaseMerger):+ strategy_value = self.strategy_or_merger.strategy if hasattr(self.strategy_or_merger, 'strategy') else "custom_merger"+ elif isinstance(self.strategy_or_merger, str):+ strategy_value = self.strategy_or_merger+ else:+ # MergeStrategy enum+ strategy_value = self.strategy_or_merger.value if hasattr(self.strategy_or_merger, 'value') else str(self.strategy_or_merger)+return {"item_schema_name": self.item_schema.__name__,- "merge_item_strategy": self.merge_item_strategy.value,+ "strategy_or_merger": strategy_value,}+ # ==================== Indexing & Query ====================++ def build_index(self) -> None:+ """Builds independent vector index for each item in the set."""+ items = self.items+ if not items:+ logger.warning("No items to index")+ return++ if self._index is not None:+ return++ documents = []+ for idx, item in enumerate(items):+ # Serialize each Item as a Document+ content = item.model_dump_json() # Use JSON string as content+ documents.append(+ Document(+ page_content=content,+ metadata={"raw": item.model_dump(), "index": idx},+ )+ )++ if documents:+ try:+ self._index = FAISS.from_documents(documents, self.embedder)+ logger.info(f"Built FAISS index with {len(documents)} items")+ except ImportError:+ logger.error("FAISS not available. Install with: pip install faiss-cpu")+ raise++ def search(self, query: str, top_k: int = 3) -> List[Item]:+ """Searches items in the set using semantic similarity.++ Args:+ query: Search query string.+ top_k: Number of results to return.++ Returns:+ List of relevant items.+ """+ if not self.items:+ logger.warning("No items to search")+ return []++ if self._index is None:+ raise Exception("Vector store not initialized")++ docs = self._index.similarity_search(query, k=top_k)+ results: List[Item] = []+ for doc in docs:+ # Attempt to restore as object+ try:+ raw = doc.metadata.get("raw", {})+ item = self.item_schema.model_validate(raw)+ results.append(item)+ except Exception as e:+ logger.warning(f"Failed to restore item: {e}")+ results.append(doc.metadata.get("raw"))++ logger.info(f"Found {len(results)} results for query: {query[:50]}...")+ return results+# ==================== Set-Specific Methods ====================def add(self, item: Item) -> None: