Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
统一使用 "temp" 目录替代 "tmp",并更新相关示例文件中的保存路径
yifanfeng97fd1195734486
diff --git a/.gitignore b/.gitignoreindex 5b11d71..c82b0b3 100644--- a/.gitignore+++ b/.gitignore@@ -1,4 +1,4 @@-tmp/+temp/logs/# Byte-compiled / optimized / DLL filesdiff --git a/examples/auto_list_demo.py b/examples/auto_list_demo.pyindex 6c0d07c..76f48ec 100644--- a/examples/auto_list_demo.py+++ b/examples/auto_list_demo.py@@ -219,7 +219,7 @@ def main():a_event_list.build_index()# 11. 保存知识print("\n[10] 保存知识到文件...")- save_path = Path(__file__).parent.parent / "tmp" / "auto_list_demo"+ save_path = Path(__file__).parent.parent / "temp" / "auto_list_demo"a_event_list.dump(str(save_path))print(f" 已保存到: {save_path}")diff --git a/examples/auto_model_demo.py b/examples/auto_model_demo.pyindex 32aa31f..d52cde2 100644--- a/examples/auto_model_demo.py+++ b/examples/auto_model_demo.py@@ -217,7 +217,7 @@ def main():# 10. 保存知识print("\n[10] 保存知识到文件...")- save_path = Path(__file__).parent.parent / "tmp" / "auto_model_demo"+ save_path = Path(__file__).parent.parent / "temp" / "auto_model_demo"a_document_model.dump(str(save_path))print(f" 已保存到: {save_path}")diff --git a/examples/auto_set_demo.py b/examples/auto_set_demo.pyindex afa7b70..0f73b66 100644--- a/examples/auto_set_demo.py+++ b/examples/auto_set_demo.py@@ -227,7 +227,7 @@ def main():print("[*] 保存怪物图鉴")print("=" * 80)- save_path = "tmp/monster_codex"+ save_path = "temp/monster_codex"monster_codex.dump(save_path)print(f"√ 已保存到:{save_path}")print(" - state.json(结构化数据)")diff --git a/hyperextract/core/base.py b/hyperextract/core/base.pyindex 395525d..85fbcc8 100644--- a/hyperextract/core/base.py+++ b/hyperextract/core/base.py@@ -81,7 +81,7 @@ class BaseAutoType(ABC, Generic[T]):self._index = None# Internal state storing the extracted knowledge- self._data: T = self._data_schema()+ # NOTE: self._data will be initialized by _init_internal_state hook (subclass responsibility)self.metadata: Dict[str, Any] = {"created_at": datetime.now(),"updated_at": datetime.now(),@@ -129,72 +129,80 @@ class BaseAutoType(ABC, Generic[T]):return self._data_schema@property+ @abstractmethoddef data(self) -> T:"""Returns all stored knowledge (read-only access).+ This is an abstract property that subclasses must implement.+ Subclasses may apply transformations to convert internal _data structure+ to the external Schema T if they differ (e.g., AutoSet converts OMem → List).+Returns:- The internal knowledge data as a Pydantic model instance.+ The internal knowledge data as a Pydantic model instance (or converted form)."""- return self._data+ pass# ==================== State Management Lifecycle Hooks ====================+ @abstractmethoddef _init_internal_state(self) -> None:"""Protected hook to initialize internal state (INIT).Called at the END of __init__ to ensure all basic attributes are set first.- Subclasses can override to initialize their own structures (e.g., _set_memory for AutoSet).+ Responsibilities:+ 1. Initialize self._data with appropriate structure (may differ from Schema T)+ 2. Initialize self._index = None++ Subclasses must implement this to set up internal structures that may be optimized+ beyond the standard Pydantic schema (e.g., dict-based for AutoSet, graph structures)."""- self._data = self._data_schema()- self._index = None+ pass+ @abstractmethoddef _set_internal_state(self, data: T) -> None:"""Protected hook to overwrite internal state (SET).Called by extract() or load() where the data provided IS the new state.Responsibilities:- 1. Replace self._data (full reset)- 2. Invalidate vector index (data changed)- 3. Subclasses: Reset auxiliary structures and fill with data+ 1. Replace self._data with new data (full reset)+ 2. Convert standard Schema T to optimized internal structure if needed+ 3. Invalidate vector index (self.clear_index())Args:data: The new data object to set."""- self._data = data- self.clear_index()+ pass+ @abstractmethoddef _update_internal_state(self, incoming_data: T) -> None:"""Protected hook to merge new data into state (UPDATE).Called by feed() where the data provided is INCREMENTAL.- Default behavior: Full merge (inefficient for large sets, good for lists).- Subclasses override for optimized incremental updates (e.g., set.add instead of merge_batch).-Responsibilities:- 1. Merge incoming_data with self._data- 2. Update self._data- 3. Invalidate vector index- 4. Subclasses: Optimized incremental update+ 1. Merge incoming_data into current internal state (optimized for incremental updates)+ 2. Invalidate vector index (self.clear_index())++ Subclasses should implement optimized incremental updates (e.g., set.add instead of+ full merge_batch, graph.add_edge instead of graph rebuild).Args:incoming_data: The incremental data to merge into the current state."""- # Default: Merge incoming with current (works but may be inefficient)- merged_data = self.merge_batch([self._data, incoming_data])- self._data = merged_data- self.clear_index()+ pass+ @abstractmethoddef _clear_internal_state(self) -> None:"""Protected hook to fully clear internal state (CLEAR).Called by clear() method.- Default: Reset to empty schema instance via _set_internal_state hook.+ Responsibilities:+ 1. Reset internal state to empty (matching _init_internal_state)"""- self._set_internal_state(self._data_schema())+ pass# ==================== Data Management ====================@@ -354,7 +362,7 @@ class BaseAutoType(ABC, Generic[T]):data = {"schema_name": self._data_schema.__name__,"data_schema": self._data_schema.model_json_schema(),- "data": self._data.model_dump(),+ "data": self.data.model_dump(),"metadata": self._prepare_metadata_for_dump(),}diff --git a/hyperextract/core/list.py b/hyperextract/core/list.pyindex c933504..2810d1a 100644--- a/hyperextract/core/list.py+++ b/hyperextract/core/list.py@@ -140,6 +140,52 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):show_progress=self.show_progress,)+ @property+ def data(self) -> AutoListSchema:+ """Returns all stored knowledge (read-only access).++ Returns:+ The internal knowledge data as AutoListSchema.+ """+ return self._data++ # ==================== State Management Lifecycle Hooks ====================++ def _init_internal_state(self) -> None:+ """+ INIT: Initialize with empty schema.+ """+ self._data = self.item_list_schema()+ self._index = None++ def _set_internal_state(self, data: AutoListSchema) -> None:+ """+ SET: Full reset. Replace with new data (e.g., load from disk).+ Called by extract() or load() where data IS the new state.+ """+ self._data = data+ self.clear_index()++ def _update_internal_state(self, incoming_data: AutoListSchema) -> None:+ """+ UPDATE: Incremental merge. Append incoming items to current list (called by feed()).++ For AutoList, incremental update means appending new items to existing list.+ """+ if incoming_data.items:+ merged_data = self.merge_batch([self._data, incoming_data])+ self._data = merged_data+ self.clear_index()++ def _clear_internal_state(self) -> None:+ """+ CLEAR: Reset to empty schema instance.+ """+ self._data = self.item_list_schema()+ self.clear_index()++ # ==================== Core Methods ====================+def merge_batch(self, data_list: List[AutoListSchema]) -> AutoListSchema:"""Pure data merge method implementing list append strategy.diff --git a/hyperextract/core/model.py b/hyperextract/core/model.pyindex a706ed1..1549a37 100644--- a/hyperextract/core/model.py+++ b/hyperextract/core/model.py@@ -79,6 +79,51 @@ class AutoModel(BaseAutoType[T]):"### Source Text:\n")+ @property+ def data(self) -> T:+ """Returns all stored knowledge (read-only access).++ Returns:+ The internal knowledge data as a Pydantic model instance.+ """+ return self._data++ # ==================== State Management Lifecycle Hooks ====================++ def _init_internal_state(self) -> None:+ """+ INIT: Initialize with empty schema instance.+ """+ self._data = self._data_schema()+ self._index = None++ def _set_internal_state(self, data: T) -> None:+ """+ SET: Full reset. Replace with new data (e.g., load from disk).+ Called by extract() or load() where data IS the new state.+ """+ self._data = data+ self.clear_index()++ def _update_internal_state(self, incoming_data: T) -> None:+ """+ UPDATE: Incremental merge. Merge fields with field-level update strategy (called by feed()).++ For AutoModel, incremental update means filling missing fields, first extraction wins.+ """+ merged_data = self.merge_batch([self._data, incoming_data])+ self._data = merged_data+ self.clear_index()++ def _clear_internal_state(self) -> None:+ """+ CLEAR: Reset to empty schema instance.+ """+ self._data = self._data_schema()+ self.clear_index()++ # ==================== Core Methods ====================+def merge_batch(self, data_list: List[T]) -> T:"""Pure data merge method implementing field-level update strategy.diff --git a/hyperextract/core/set.py b/hyperextract/core/set.pyindex 1e9f036..3ae957c 100644--- a/hyperextract/core/set.py+++ b/hyperextract/core/set.py@@ -82,7 +82,6 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):max_workers: int = 10,show_progress: bool = True,llm_batch_size: int = 10,- **kwargs,):"""Initialize AutoSet with key extractor and merge strategy.@@ -98,7 +97,6 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):max_workers: Maximum concurrent extraction tasks.show_progress: Whether to log progress information.llm_batch_size: Batch size for LLM merge operations (for LLM_MERGE strategy).- **kwargs: Additional arguments passed to parent class."""# Store item_schema@@ -114,7 +112,6 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):),)- # Call BaseAutoType.__init__ (not ListKnowledge)super().__init__(data_schema=self.item_set_schema,llm_client=llm_client,@@ -124,7 +121,6 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):chunk_overlap=chunk_overlap,max_workers=max_workers,show_progress=show_progress,- **kwargs,)# AutoSet-specific attributes (before _init_internal_state is called by parent)@@ -141,12 +137,12 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):)# Initialize OMem instance for storage and deduplication- self._set_memory: OMem[Item] = OMem(+ self._data: OMem[Item] = OMem(memory_schema=item_schema,key_extractor=key_extractor,llm_client=llm_client,embedder=embedder,- merge_strategy=self._merger,+ strategy_or_merger=self._merger,verbose=show_progress,)@@ -184,7 +180,14 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):"### Source Text:\n")- # ==================== Properties ====================+ @property+ def data(self) -> AutoSetSchema[Item]:+ """Returns all stored knowledge (read-only access).++ Returns:+ The internal knowledge data as a Pydantic model instance.+ """+ return self.data_schema(items=self.items)@propertydef items(self) -> List[Item]:@@ -193,7 +196,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):Returns:List of unique items."""- return self._set_memory.items+ return self._data.items@propertydef keys(self) -> List[Any]:@@ -202,11 +205,11 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):Returns:List of unique key values."""- return self._set_memory.keys+ return self._data.keysdef __len__(self) -> int:"""Returns the number of unique items in the set."""- return len(self._set_memory.items)+ return len(self._data)def __contains__(self, key: Any) -> bool:"""Checks if a unique key exists in the set.@@ -217,7 +220,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):Returns:True if key exists, False otherwise."""- return self._set_memory.get(key) is not None+ return self._data.get(key) is not Nonedef __repr__(self) -> str:"""Returns a developer-friendly representation."""@@ -244,48 +247,39 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):def _init_internal_state(self) -> None:"""- INIT: Initialize OMem as empty (called at end of __init__).+ INIT: Initialize OMem as empty."""- if hasattr(self, '_set_memory'):- self._set_memory.clear()- super()._init_internal_state()+ self._data.clear()+ self.clear_index()def _set_internal_state(self, data: AutoSetSchema[Item]) -> None:"""SET: Full Reset. Wipe OMem and refill from data (e.g., load from disk).Called by extract() or load() where data IS the new state."""- if hasattr(self, '_set_memory'):- self._set_memory.clear()- if data.items:- self._set_memory.add(data.items)-- super()._set_internal_state(data)+ self._data.clear()+ if data.items:+ self._data.add(data.items)+ self.clear_index()def _update_internal_state(self, incoming_data: AutoSetSchema[Item]) -> None:"""UPDATE: Incremental merge. Add to OMem efficiently (called by feed()).-+Unlike the default behavior which uses merge_batch for full re-merge,AutoSet optimizes this by directly adding items to OMem, whichhandles deduplication and merging internally."""- if hasattr(self, '_set_memory') and incoming_data.items:- # OMem handles diff/merge efficiently with configured merge strategy- self._set_memory.add(incoming_data.items)-- # Sync state: Create snapshot from OMem- # We bypass super()._update_internal_state() to avoid redundant merge_batch- self._data = self.item_set_schema(items=self._set_memory.items)+ if incoming_data.items:+ self._data.add(incoming_data.items)self.clear_index()def _clear_internal_state(self) -> None:"""CLEAR: Wipe OMem (called by clear())."""- if hasattr(self, '_set_memory'):- self._set_memory.clear()- super()._clear_internal_state()+ self._data.clear()+ self.clear_index()# ==================== Core Override Methods ====================@@ -302,7 +296,6 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):Returns:New merged AutoSetSchema with deduplicated items and resolved conflicts."""- # Step 1: Collect all items from all containersall_items = []for data in data_list:all_items.extend(data.items)@@ -314,33 +307,13 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):f"Merging {len(all_items)} items from {len(data_list)} containers...")- # Step 2: Create temporary OMem instance for merging- temp_merger = create_merger(- strategy=self.merge_item_strategy,- key_extractor=self.key_extractor,- llm_client=self.llm_client,- item_schema=self.item_schema,- )-- temp_omem: OMem[Item] = OMem(- memory_schema=self.item_schema,- key_extractor=self.key_extractor,- llm_client=self.llm_client,- embedder=self.embedder,- merge_strategy=temp_merger,- verbose=self.show_progress,- )-- # Step 3: Add all items to temp OMem (automatic dedup and merge)- temp_omem.add(all_items)- merged_items = temp_omem.items+ merged_items = self._merger.merge(all_items)logger.info(f"Merged into {len(merged_items)} unique items "f"(strategy: {self.merge_item_strategy.value})")- # Step 4: Return new container (does not modify self)return self.item_set_schema(items=merged_items)# ==================== Serialization Helpers ====================@@ -365,10 +338,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):item: The item to add."""# OMem handles deduplication and merging automatically- self._set_memory.add([item])-- # Sync with _data- self._data = self.item_set_schema(items=self._set_memory.items)+ self._data.add([item])self.clear_index()self.metadata["updated_at"] = datetime.now()@@ -382,14 +352,13 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):The removed item, or None if not found."""# Get item before removing- item = self._set_memory.get(key)+ item = self._data.get(key)if item is None:return None-+# Remove from OMem- removed = self._set_memory.remove(key)+ removed = self._data.remove(key)if removed:- self._data = self.item_set_schema(items=self._set_memory.items)self.clear_index()self.metadata["updated_at"] = datetime.now()logger.debug(f"Removed item with key '{key}'")@@ -404,7 +373,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):Returns:True if key exists, False otherwise."""- return self._set_memory.get(key) is not None+ return self._data.get(key) is not Nonedef get(self, key: Any, default: Optional[Item] = None) -> Optional[Item]:"""Gets an item by its unique key value.@@ -416,7 +385,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):Returns:The item if found, otherwise default."""- result = self._set_memory.get(key)+ result = self._data.get(key)return result if result is not None else defaultdef update(self, items: List[Item]) -> None:@@ -444,9 +413,8 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):- Updates metadata timestamp"""try:- if self._set_memory.get(key) is not None:- self._set_memory.remove(key)- self._data = self.item_set_schema(items=self._set_memory.items)+ if self._data.get(key) is not None:+ self._data.remove(key)self.clear_index()self.metadata["updated_at"] = datetime.now()except (KeyError, Exception):@@ -472,14 +440,13 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):- Clears the vector index (needs rebuild)- Updates metadata timestamp"""- if not self._set_memory.items:+ if not self._data.items:raise KeyError("pop from an empty AutoSet")- item = self._set_memory.items[0]+ item = self._data.items[0]key = self.key_extractor(item)- self._set_memory.remove(key)-- self._data = self.item_set_schema(items=self._set_memory.items)+ self._data.remove(key)+self.clear_index()self.metadata["updated_at"] = datetime.now()@@ -497,15 +464,10 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):>>> # Original skills unchanged"""new_set = self._create_empty_instance()-+# Add copies of all items- items_copy = [item.model_copy(deep=True) for item in self._set_memory.items]- new_set._set_memory.add(items_copy)-- # Sync state- new_data = new_set.item_set_schema(items=new_set._set_memory.items)- new_set._data = new_data- new_set.clear_index()+ items_copy = [item.model_copy(deep=True) for item in self._data.items]+ new_set._data.add(items_copy)new_set.metadata = self.metadata.copy()new_set.metadata["created_at"] = datetime.now()@@ -537,14 +499,10 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):# Create new instancenew_set = self._create_empty_instance()-- # Add all items from both sets (OMem handles merge automatically)- all_items = self._set_memory.items + other._set_memory.items- new_set._set_memory.add(all_items)- merged_data = new_set.item_set_schema(items=new_set._set_memory.items)- new_set._data = merged_data- new_set.clear_index()+ # Add all items from both sets (OMem handles merge automatically)+ all_items = self._data.items + other._data.items+ new_set._data.add(all_items)new_set.metadata["updated_at"] = datetime.now()return new_set@@ -575,18 +533,13 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):new_set = self._create_empty_instance()# Only keep items present in both sets (by key)- other_keys = {self.key_extractor(item) for item in other._set_memory.items}intersection_items = [- item for item in self._set_memory.items- if self.key_extractor(item) in other_keys+ item for item in self._data.items if self.key_extractor(item) in other.keys]-+if intersection_items:- new_set._set_memory.add(intersection_items)+ new_set._data.add(intersection_items)- merged_data = new_set.item_set_schema(items=new_set._set_memory.items)- new_set._data = merged_data- new_set.clear_index()new_set.metadata["updated_at"] = datetime.now()return new_set@@ -617,18 +570,15 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):new_set = self._create_empty_instance()# Only keep items not in other (by key)- other_keys = {self.key_extractor(item) for item in other._set_memory.items}difference_items = [- item for item in self._set_memory.items- if self.key_extractor(item) not in other_keys+ item+ for item in self._data.items+ if self.key_extractor(item) not in other.keys]-+if difference_items:- new_set._set_memory.add(difference_items)+ new_set._data.add(difference_items)- merged_data = new_set.item_set_schema(items=new_set._set_memory.items)- new_set._data = merged_data- new_set.clear_index()new_set.metadata["updated_at"] = datetime.now()return new_set@@ -658,28 +608,25 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):# Create new instancenew_set = self._create_empty_instance()- # Get keys- self_keys = {self.key_extractor(item) for item in self._set_memory.items}- other_keys = {self.key_extractor(item) for item in other._set_memory.items}-# Items only in selfsymmetric_items = [- item for item in self._set_memory.items- if self.key_extractor(item) not in other_keys+ item+ for item in self._data.items+ if self.key_extractor(item) not in other.keys]-+# Add items only in other- symmetric_items.extend([- item for item in other._set_memory.items- if self.key_extractor(item) not in self_keys- ])-+ symmetric_items.extend(+ [+ item+ for item in other._data.items+ if self.key_extractor(item) not in self.keys+ ]+ )+if symmetric_items:- new_set._set_memory.add(symmetric_items)+ new_set._data.add(symmetric_items)- merged_data = new_set.item_set_schema(items=new_set._set_memory.items)- new_set._data = merged_data- new_set.clear_index()new_set.metadata["updated_at"] = datetime.now()return new_set@@ -725,9 +672,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):if self.item_schema != other.item_schema:return False- self_keys = {self.key_extractor(item) for item in self._set_memory.items}- other_keys = {self.key_extractor(item) for item in other._set_memory.items}- return self_keys == other_keys+ return self.keys == other.keysdef __ne__(self, other: Any) -> bool:"""Inequality comparison: set1 != set2.@@ -761,9 +706,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):f"Left: {self.item_schema.__name__}, Right: {other.item_schema.__name__}")- self_keys = {self.key_extractor(item) for item in self._set_memory.items}- other_keys = {self.key_extractor(item) for item in other._set_memory.items}- return self_keys.issubset(other_keys)+ return set(self.keys).issubset(set(other.keys))def __lt__(self, other: "AutoSet[Item]") -> bool:"""Proper subset comparison: set1 < set2.@@ -803,9 +746,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):f"Left: {self.item_schema.__name__}, Right: {other.item_schema.__name__}")- self_keys = {self.key_extractor(item) for item in self._set_memory.items}- other_keys = {self.key_extractor(item) for item in other._set_memory.items}- return self_keys.issuperset(other_keys)+ return set(self.keys).issuperset(set(other.keys))def __gt__(self, other: "AutoSet[Item]") -> bool:"""Proper superset comparison: set1 > set2.@@ -878,6 +819,4 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):f"Left: {self.item_schema.__name__}, Right: {other.item_schema.__name__}")- self_keys = {self.key_extractor(item) for item in self._set_memory.items}- other_keys = {self.key_extractor(item) for item in other._set_memory.items}- return self_keys.isdisjoint(other_keys)+ return set(self.keys).isdisjoint(set(other.keys))