Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
重构数据合并方法,统一命名为 merge_batch_data,增强代码一致性和可读性;添加索引存储和加载功能
yifanfeng979c0ca975e61f
diff --git a/hyperextract/core/base.py b/hyperextract/core/base.pyindex e6aaf60..77842db 100644--- a/hyperextract/core/base.py+++ b/hyperextract/core/base.py@@ -65,13 +65,6 @@ class BaseAutoType(ABC, Generic[T]):self.max_workers = max_workersself.show_progress = show_progress- # Initialize prompt template and LLM chain for structured extraction- self.prompt_template = ChatPromptTemplate.from_template(- f"{self.prompt}{{chunk_text}}"- )- self.llm_with_schema = self.llm_client.with_structured_output(self._data_schema)- self.llm_chain_extract = self.prompt_template | self.llm_with_schema-# Initialize text splitter for chunking long documentsself.text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size,@@ -240,17 +233,25 @@ class BaseAutoType(ABC, Generic[T]):Returns:The extracted knowledge data."""++ prompt_template = ChatPromptTemplate.from_template(+ f"{self.prompt}{{chunk_text}}"+ )+ llm_chain = prompt_template | self.llm_client.with_structured_output(+ self._data_schema+ )+if len(text) <= self.chunk_size:- extracted_data = self.llm_chain_extract.invoke({"chunk_text": text})+ extracted_data = llm_chain.invoke({"chunk_text": text})extracted_data_list = [extracted_data]else:chunks = self.text_splitter.split_text(text)inputs = [{"chunk_text": chunk} for chunk in chunks]- extracted_data_list = self.llm_chain_extract.batch(+ extracted_data_list = llm_chain.batch(inputs, config={"max_concurrency": self.max_workers})- merged_data = self.merge_batch(extracted_data_list)+ merged_data = self.merge_batch_data(extracted_data_list)return merged_datadef extract(self, text: str) -> "BaseAutoType[T]":@@ -275,12 +276,12 @@ class BaseAutoType(ABC, Generic[T]):return new_instance- def feed(self, text: str) -> "BaseAutoType[T]":+ def feed_text(self, text: str) -> "BaseAutoType[T]":"""Ingests text into the CURRENT knowledge base instance.This modifies the internal state by merging new data with existing data.- Supports method chaining (e.g., kb.feed(text1).feed(text2)).+ Supports method chaining (e.g., kb.feed_text(text1).feed_text(text2)).Args:text: Input text.@@ -298,7 +299,7 @@ class BaseAutoType(ABC, Generic[T]):return self@abstractmethod- def merge_batch(self, data_list: List[T]) -> T:+ def merge_batch_data(self, data_list: List[T]) -> T:"""Merges multiple knowledge data objects into a single unified object.This is a pure data transformation method that does not modify internal state.@@ -355,151 +356,160 @@ class BaseAutoType(ABC, Generic[T]):def dump(self, folder_path: str | Path) -> None:"""Exports knowledge to a specified folder.- Saves to the folder:- 1. Structured data (self._data) - saved as state.json- 2. Vector index (self._index) - saved via _dump_index_storage hook+ Storage Structure:+ folder/+ ├── config.json # Metadata, Schema Info, Parameters+ ├── data.json # Pure Knowledge Data+ └── index/ # Vector Index FilesArgs:folder_path: Target folder path for saving."""folder = Path(folder_path)- if folder.exists() and folder.is_file():- raise Exception("Folder path is a file, please provide a folder path.")-- if not folder.exists():- folder.mkdir(parents=True, exist_ok=True)+ if folder.is_file():+ raise FileExistsError(f"Path is a file, expected directory: {folder}")+ folder.mkdir(parents=True, exist_ok=True)- try:- # 1. Save structured data- data = {- "schema_name": self._data_schema.__name__,- "data_schema": self._data_schema.model_json_schema(),- "data": self.data.model_dump(),- "metadata": self._prepare_metadata_for_dump(),- }+ # 1. Save Configuration (Schema, Metadata, Parameters)+ self._save_config(folder)- # Allow subclasses to add extra fields- data.update(self._get_extra_dump_data())+ # 2. Save Data (Pure Data)+ self._save_data(folder)- json_str = json.dumps(data, indent=2, ensure_ascii=False, default=str)- data_file = folder / "state.json"- with open(data_file, "w", encoding="utf-8") as f:- f.write(json_str)-- # 2. Save vector index (via hook for flexibility)- if self._index is not None:- self._dump_index_storage(folder)-- except Exception:- raise+ # 3. Save Index (Optional)+ if self._index is not None:+ self._dump_index_storage(folder)def load(self, folder_path: str | Path) -> None:"""Loads knowledge from a specified folder.- Loads from the folder:- 1. Structured data (self._data) - loaded from state.json- 2. Vector index (self._index) - loaded via _load_index_storage hook-Args:folder_path: Source folder path containing saved knowledge."""folder = Path(folder_path)if not folder.is_dir():- raise ValueError(f"Folder does not exist: {folder_path}")+ raise FileNotFoundError(f"Folder not found: {folder_path}")++ # 1. Load Configuration (Validate compatibility)+ self._load_config(folder)- # 1. Load structured data- data_file = folder / "state.json"- if not data_file.is_file():- raise ValueError(f"Data file not found: {data_file}")+ # 2. Load Data (Restore state)+ self._load_data(folder)- with open(data_file, "r", encoding="utf-8") as f:- data = json.load(f)+ # 3. Load Index (Restore vector search)+ self._load_index_storage(folder)- # Load data using hook for proper state sync- loaded_data = self._data_schema.model_validate(data.get("data", {}))- self._set_data_state(loaded_data)+ # ==================== Serialization: Components ====================++ def _save_config(self, folder: Path) -> None:+ """Saves metadata, schema info, and instance parameters."""+ config = {+ "version": "1.0",+ "type": self.__class__.__name__,+ "schema": self._data_schema.__name__,+ "params": {+ "prompt": self.prompt,+ "chunk_size": self.chunk_size,+ "chunk_overlap": self.chunk_overlap,+ "max_workers": self.max_workers,+ },+ "metadata": {+ k: str(v) if isinstance(v, datetime) else v+ for k, v in self.metadata.items()+ },+ "extra": self._get_extra_config_data(),+ }- # Update metadata with loaded values- if "metadata" in data:- self.metadata.update(data["metadata"])- self.metadata["updated_at"] = datetime.now()+ with open(folder / "config.json", "w", encoding="utf-8") as f:+ json.dump(config, f, indent=2, ensure_ascii=False)- # Allow subclasses to load extra data- self._load_extra_data(data)+ def _save_data(self, folder: Path) -> None:+ """Saves the pure knowledge data."""+ export_data = self.data.model_dump()- # 2. Load vector index (via hook for flexibility)- self._load_index_storage(folder)+ with open(folder / "data.json", "w", encoding="utf-8") as f:+ json.dump(export_data, f, indent=2, ensure_ascii=False, default=str)- # ==================== Serialization: Index Storage Hooks ====================+ def _load_config(self, folder: Path) -> None:+ """Loads and validates configuration."""+ config_path = folder / "config.json"+ if not config_path.exists():+ raise FileNotFoundError(f"Config file not found: {config_path}")+ with open(config_path, "r", encoding="utf-8") as f:+ config = json.load(f)++ # Strict schema validation - raise error if mismatch+ if config.get("schema") != self._data_schema.__name__:+ raise ValueError(+ f"Schema mismatch: Expected '{self._data_schema.__name__}', "+ f"but found '{config.get('schema')}' in config.json"+ )++ # Restore metadata+ if "metadata" in config:+ self.metadata.update(config["metadata"])++ # Restore extra config+ if "extra" in config:+ self._load_extra_config_data(config["extra"])++ def _load_data(self, folder: Path) -> None:+ """Loads data and restores internal state."""+ data_path = folder / "data.json"+ if not data_path.exists():+ raise FileNotFoundError(f"Data file not found: {data_path}")++ with open(data_path, "r", encoding="utf-8") as f:+ raw_data = json.load(f)++ # Validate and Set State+ validated_data = self._data_schema.model_validate(raw_data)+ self._set_data_state(validated_data)++ @abstractmethoddef _dump_index_storage(self, folder: Path) -> None:"""HOOK: Save vector index to disk.- Default Implementation: FAISS save_local-- Subclasses can override to support:- - Alternative vector store implementations (Chroma, Pinecone, etc.)- - Custom serialization logic- - Multiple indices+ Subclasses must implement to support their specific vector store+ (FAISS, Chroma, Pinecone, etc.).Args:folder: Target folder for saving index files."""- index_path = str(folder / "faiss_index")- self._index.save_local(index_path)+ pass+ @abstractmethoddef _load_index_storage(self, folder: Path) -> None:"""HOOK: Load vector index from disk.- Default Implementation: FAISS load_local-- Subclasses can override to support:- - Alternative vector store implementations (Chroma, Pinecone, etc.)- - Custom deserialization logic- - Multiple indices+ Subclasses must implement to support their specific vector store+ (FAISS, Chroma, Pinecone, etc.).Args:folder: Source folder containing index files."""- from langchain_community.vectorstores import FAISS-- index_path = str(folder / "faiss_index")- if Path(index_path).exists():- try:- self._index = FAISS.load_local(- index_path, self.embedder, allow_dangerous_deserialization=True- )- except Exception:- self._index = None- else:- self._index = None-- # ==================== Serialization: Helper Methods ====================+ pass- def _prepare_metadata_for_dump(self) -> Dict[str, Any]:- """Helper to serialize metadata values (e.g., datetimes)."""- return {- k: str(v) if isinstance(v, datetime) else v- for k, v in self.metadata.items()- }+ # ==================== Serialization: Hooks ====================- def _get_extra_dump_data(self) -> Dict[str, Any]:- """Hook for subclasses to add extra fields to state.json.+ def _get_extra_config_data(self) -> Dict[str, Any]:+ """Hook to save subclass-specific configuration (not data).- Override this method to save subclass-specific data (e.g., merge strategy).+ Override this method to save subclass-specific config like merge strategy.Returns:- Dictionary of extra data to include in state.json.+ Dictionary of extra config data."""return {}- def _load_extra_data(self, json_data: Dict[str, Any]) -> None:- """Hook for subclasses to extract extra fields from state.json.+ def _load_extra_config_data(self, extra_config: Dict[str, Any]) -> None:+ """Hook to load subclass-specific configuration.- Override this method to restore subclass-specific data.+ Override this method to restore subclass-specific config.Args:- json_data: The loaded JSON data dictionary.+ extra_config: Extra configuration data from config.json."""pass@@ -545,7 +555,7 @@ class BaseAutoType(ABC, Generic[T]):)# Merge the data from both instances- merged_data = self.merge_batch([self._data, other._data])+ merged_data = self.merge_batch_data([self._data, other._data])# Create a new instance with the same configurationnew_instance = self._create_empty_instance()diff --git a/hyperextract/core/list.py b/hyperextract/core/list.pyindex 6c88478..a3df171 100644--- a/hyperextract/core/list.py+++ b/hyperextract/core/list.py@@ -1,5 +1,6 @@"""List Knowledge Pattern - extracts a collection of objects from text."""+from pathlib import Pathfrom typing import (List,Any,@@ -69,7 +70,7 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):chunk_overlap: int = 200,max_workers: int = 10,show_progress: bool = True,- **kwargs,+ fields_for_index: List[str] | None = None,):"""Initialize AutoList with item schema and configuration.@@ -82,8 +83,11 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):chunk_overlap: Overlapping characters between adjacent chunks.max_workers: Maximum concurrent extraction tasks.show_progress: Whether to log progress information.+ fields_for_index: Optional list of field names to include in vector index.+ If None, all fields are indexed."""self.item_schema = item_schema+ self.fields_for_index = fields_for_indexcontainer_name = f"{item_schema.__name__}List"self.item_list_schema = create_model(@@ -103,7 +107,6 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):chunk_overlap=chunk_overlap,max_workers=max_workers,show_progress=show_progress,- **kwargs,)def _default_prompt(self) -> str:@@ -138,6 +141,7 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):chunk_overlap=self.chunk_overlap,max_workers=self.max_workers,show_progress=self.show_progress,+ fields_for_index=self.fields_for_index,)@property@@ -158,10 +162,6 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):"""self._data = self.item_list_schema()- def _init_index_state(self) -> None:- """Initialize vector index to empty state."""- self._index = None-def _set_data_state(self, data: AutoListSchema) -> None:"""SET: Full reset. Replace with new data (e.g., load from disk).@@ -177,13 +177,17 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):For AutoList, incremental update means appending new items to existing list."""if incoming_data.items:- merged_data = self.merge_batch([self._data, incoming_data])+ merged_data = self.merge_batch_data([self._data, incoming_data])self._data = merged_dataself.clear_index()+ def _init_index_state(self) -> None:+ """Initialize vector index to empty state."""+ self._index = None+# ==================== Core Methods ====================- def merge_batch(self, data_list: List[AutoListSchema]) -> AutoListSchema:+ def merge_batch_data(self, data_list: List[AutoListSchema]) -> AutoListSchema:"""Pure data merge method implementing list append strategy.Merge strategy: Collects all items from all container objects and merges them@@ -199,16 +203,17 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):"""all_items = []- # Collect all items from each containerfor data in data_list:- copied_data = data.model_copy(deep=True)- all_items.extend(copied_data.items)+ all_items.extend(data.items)- # Return a new container objectreturn self.item_list_schema(items=all_items)def build_index(self) -> None:- """Builds independent vector index for each item in the list."""+ """Builds independent vector index for each item in the list.++ If fields_for_index is specified, only those fields are indexed.+ Otherwise, all fields are indexed.+ """items = self.itemsif not items:logger.warning("No items to index")@@ -219,8 +224,16 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):documents = []for idx, item in enumerate(items):- # Serialize each Item as a Document- content = item.model_dump_json() # Use JSON string as content+ # Extract content based on fields_for_index+ if self.fields_for_index:+ # Only index specified fields+ item_dict = item.model_dump()+ indexed_fields = {k: v for k, v in item_dict.items() if k in self.fields_for_index}+ content = str(indexed_fields)+ else:+ # Index all fields+ content = item.model_dump_json()+documents.append(Document(page_content=content,@@ -231,7 +244,7 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):if documents:try:self._index = FAISS.from_documents(documents, self.embedder)- logger.info(f"Built FAISS index with {len(documents)} items")+ logger.info(f"Built FAISS index with {len(documents)} items (fields: {self.fields_for_index or 'all'})")except ImportError:logger.error("FAISS not available. Install with: pip install faiss-cpu")raise@@ -268,6 +281,30 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):logger.info(f"Found {len(results)} results for query: {query[:50]}...")return results+ # ==================== Index Storage ====================++ def _dump_index_storage(self, folder: Path) -> None:+ """Saves FAISS vector index to disk."""+ if self._index is None:+ return+ index_path = str(folder / "index")+ self._index.save_local(index_path)++ def _load_index_storage(self, folder: Path) -> None:+ """Loads FAISS vector index from disk."""+ from langchain_community.vectorstores import FAISS++ index_path = folder / "index"+ if index_path.exists():+ try:+ self._index = FAISS.load_local(+ str(index_path), self.embedder, allow_dangerous_deserialization=True+ )+ except Exception:+ self._index = None+ else:+ self._index = None+# ==================== Pythonic Sequence Operations ====================def __len__(self) -> int:@@ -710,11 +747,6 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):self._data.items.reverse()self.metadata["updated_at"] = datetime.now()- # Rebuild index if it exists to maintain consistency- if self._index is not None:- self._index = None- self.build_index()-def sort(self, key: Union[Callable[[Item], Any], None] = None, reverse: bool = False) -> None:@@ -749,11 +781,6 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):self._data.items.sort(key=key, reverse=reverse)self.metadata["updated_at"] = datetime.now()- # Rebuild index if it exists to maintain consistency- if self._index is not None:- self._index = None- self.build_index()-# ==================== Helper Methods ====================def _validate_item_schema(self, item: Any) -> None:diff --git a/hyperextract/core/model.py b/hyperextract/core/model.pyindex eac53b9..17fcdf6 100644--- a/hyperextract/core/model.py+++ b/hyperextract/core/model.py@@ -1,5 +1,6 @@"""Unit Knowledge Pattern - extracts a single structured object from text."""+from pathlib import Pathfrom typing import List, Any, Typefrom datetime import datetimefrom langchain_core.language_models.chat_models import BaseChatModel@@ -115,13 +116,13 @@ class AutoModel(BaseAutoType[T]):For AutoModel, incremental update means filling missing fields, first extraction wins."""- merged_data = self.merge_batch([self._data, incoming_data])+ merged_data = self.merge_batch_data([self._data, incoming_data])self._data = merged_dataself.clear_index()# ==================== Core Methods ====================- def merge_batch(self, data_list: List[T]) -> T:+ def merge_batch_data(self, data_list: List[T]) -> T:"""Pure data merge method implementing field-level update strategy.Merge strategy: First extraction takes precedence. Subsequent extractions only fill@@ -211,6 +212,30 @@ class AutoModel(BaseAutoType[T]):logger.info(f"Found {len(results)} results for query: {query[:50]}...")return results+ # ==================== Index Storage ====================++ def _dump_index_storage(self, folder: Path) -> None:+ """Saves FAISS vector index to disk."""+ if self._index is None:+ return+ index_path = str(folder / "index")+ self._index.save_local(index_path)++ def _load_index_storage(self, folder: Path) -> None:+ """Loads FAISS vector index from disk."""+ from langchain_community.vectorstores import FAISS++ index_path = folder / "index"+ if index_path.exists():+ try:+ self._index = FAISS.load_local(+ str(index_path), self.embedder, allow_dangerous_deserialization=True+ )+ except Exception:+ self._index = None+ else:+ self._index = None+# ==================== Operators ====================def __add__(self, other):diff --git a/hyperextract/core/set.py b/hyperextract/core/set.pyindex 9038bf4..686aec8 100644--- a/hyperextract/core/set.py+++ b/hyperextract/core/set.py@@ -4,7 +4,18 @@ Provides automatic deduplication based on a user-specified unique key field.Supports multiple merge strategies including LLM-powered intelligent merging."""-from typing import List, Any, Type, TypeVar, Generic, Optional, Callable, Union+from pathlib import Path+from typing import (+ List,+ Any,+ Type,+ TypeVar,+ Generic,+ Optional,+ Callable,+ Union,+ TYPE_CHECKING,+)from datetime import datetimefrom ontomem import OMemfrom ontomem.merger import MergeStrategy, create_merger, BaseMerger@@ -15,13 +26,7 @@ from langchain_core.documents import Documentfrom langchain_community.vectorstores import FAISSfrom .base import BaseAutoType--try:- from hyperextract.config import logger-except ImportError:- import logging-- logger = logging.getLogger(__name__)+from hyperextract.config import loggerItem = TypeVar("Item", bound=BaseModel)@@ -70,6 +75,10 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):1"""+ if TYPE_CHECKING:+ # Use generic version during type checking to maintain complete type hints+ item_set_schema: Type[AutoSetSchema[Item]]+def __init__(self,item_schema: Type[Item],@@ -77,7 +86,9 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):embedder: Embeddings,*,key_extractor: Callable[[Item], Any],- strategy_or_merger: Union[MergeStrategy, BaseMerger] = MergeStrategy.LLM.PREFER_INCOMING,+ strategy_or_merger: Union[+ MergeStrategy, BaseMerger+ ] = MergeStrategy.LLM.PREFER_INCOMING,prompt: str = "",chunk_size: int = 2000,chunk_overlap: int = 200,@@ -130,7 +141,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):if kwargs:logger.warning("Initialized with a Merger instance. Additional kwargs are ignored: %s",- list(kwargs.keys())+ list(kwargs.keys()),)self._merger = strategy_or_mergerelse:@@ -298,7 +309,9 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):# ==================== Core Override Methods ====================- def merge_batch(self, data_list: List[AutoSetSchema[Item]]) -> AutoSetSchema[Item]:+ def merge_batch_data(+ self, data_list: List[AutoSetSchema[Item]]+ ) -> AutoSetSchema[Item]:"""Merges multiple data containers with automatic deduplication.Pure function: Does not modify internal state.@@ -326,13 +339,20 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):# Get strategy value for loggingif isinstance(self.strategy_or_merger, BaseMerger):- strategy_str = self.strategy_or_merger.strategy if hasattr(self.strategy_or_merger, 'strategy') else "custom_merger"+ strategy_str = (+ self.strategy_or_merger.strategy+ if hasattr(self.strategy_or_merger, "strategy")+ else "custom_merger"+ )else:- strategy_str = self.strategy_or_merger.value if hasattr(self.strategy_or_merger, 'value') else str(self.strategy_or_merger)+ strategy_str = (+ self.strategy_or_merger.value+ if hasattr(self.strategy_or_merger, "value")+ else str(self.strategy_or_merger)+ )logger.info(- f"Merged into {len(merged_items)} unique items "- f"(strategy: {strategy_str})"+ f"Merged into {len(merged_items)} unique items (strategy: {strategy_str})")return self.item_set_schema(items=merged_items)@@ -347,13 +367,21 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):"""# Get strategy value: handle different types of strategy_or_mergerif isinstance(self.strategy_or_merger, BaseMerger):- strategy_value = self.strategy_or_merger.strategy if hasattr(self.strategy_or_merger, 'strategy') else "custom_merger"+ strategy_value = (+ self.strategy_or_merger.strategy+ if hasattr(self.strategy_or_merger, "strategy")+ else "custom_merger"+ )elif isinstance(self.strategy_or_merger, str):strategy_value = self.strategy_or_mergerelse:# MergeStrategy enum- strategy_value = self.strategy_or_merger.value if hasattr(self.strategy_or_merger, 'value') else str(self.strategy_or_merger)-+ strategy_value = (+ self.strategy_or_merger.value+ if hasattr(self.strategy_or_merger, "value")+ else str(self.strategy_or_merger)+ )+return {"item_schema_name": self.item_schema.__name__,"strategy_or_merger": strategy_value,@@ -422,6 +450,30 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):logger.info(f"Found {len(results)} results for query: {query[:50]}...")return results+ # ==================== Index Storage ====================++ def _dump_index_storage(self, folder: Path) -> None:+ """Saves FAISS vector index to disk."""+ if self._index is None:+ return+ index_path = str(folder / "index")+ self._index.save_local(index_path)++ def _load_index_storage(self, folder: Path) -> None:+ """Loads FAISS vector index from disk."""+ from langchain_community.vectorstores import FAISS++ index_path = folder / "index"+ if index_path.exists():+ try:+ self._index = FAISS.load_local(+ str(index_path), self.embedder, allow_dangerous_deserialization=True+ )+ except Exception:+ self._index = None+ else:+ self._index = None+# ==================== Set-Specific Methods ====================def add(self, item: Item) -> None: