Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

重构数据合并方法,统一命名为 merge_batch_data,增强代码一致性和可读性;添加索引存储和加载功能

yifanfeng979c0ca975e61f
diff --git a/hyperextract/core/base.py b/hyperextract/core/base.py
index e6aaf60..77842db 100644
--- a/hyperextract/core/base.py
+++ b/hyperextract/core/base.py
@@ -65,13 +65,6 @@ class BaseAutoType(ABC, Generic[T]):
self.max_workers = max_workers
self.show_progress = show_progress
- # Initialize prompt template and LLM chain for structured extraction
- self.prompt_template = ChatPromptTemplate.from_template(
- f"{self.prompt}{{chunk_text}}"
- )
- self.llm_with_schema = self.llm_client.with_structured_output(self._data_schema)
- self.llm_chain_extract = self.prompt_template | self.llm_with_schema
-
# Initialize text splitter for chunking long documents
self.text_splitter = RecursiveCharacterTextSplitter(
chunk_size=chunk_size,
@@ -240,17 +233,25 @@ class BaseAutoType(ABC, Generic[T]):
Returns:
The extracted knowledge data.
"""
+
+ prompt_template = ChatPromptTemplate.from_template(
+ f"{self.prompt}{{chunk_text}}"
+ )
+ llm_chain = prompt_template | self.llm_client.with_structured_output(
+ self._data_schema
+ )
+
if len(text) <= self.chunk_size:
- extracted_data = self.llm_chain_extract.invoke({"chunk_text": text})
+ extracted_data = llm_chain.invoke({"chunk_text": text})
extracted_data_list = [extracted_data]
else:
chunks = self.text_splitter.split_text(text)
inputs = [{"chunk_text": chunk} for chunk in chunks]
- extracted_data_list = self.llm_chain_extract.batch(
+ extracted_data_list = llm_chain.batch(
inputs, config={"max_concurrency": self.max_workers}
)
- merged_data = self.merge_batch(extracted_data_list)
+ merged_data = self.merge_batch_data(extracted_data_list)
return merged_data
def extract(self, text: str) -> "BaseAutoType[T]":
@@ -275,12 +276,12 @@ class BaseAutoType(ABC, Generic[T]):
return new_instance
- def feed(self, text: str) -> "BaseAutoType[T]":
+ def feed_text(self, text: str) -> "BaseAutoType[T]":
"""
Ingests text into the CURRENT knowledge base instance.
This modifies the internal state by merging new data with existing data.
- Supports method chaining (e.g., kb.feed(text1).feed(text2)).
+ Supports method chaining (e.g., kb.feed_text(text1).feed_text(text2)).
Args:
text: Input text.
@@ -298,7 +299,7 @@ class BaseAutoType(ABC, Generic[T]):
return self
@abstractmethod
- def merge_batch(self, data_list: List[T]) -> T:
+ def merge_batch_data(self, data_list: List[T]) -> T:
"""Merges multiple knowledge data objects into a single unified object.
This is a pure data transformation method that does not modify internal state.
@@ -355,151 +356,160 @@ class BaseAutoType(ABC, Generic[T]):
def dump(self, folder_path: str | Path) -> None:
"""Exports knowledge to a specified folder.
- Saves to the folder:
- 1. Structured data (self._data) - saved as state.json
- 2. Vector index (self._index) - saved via _dump_index_storage hook
+ Storage Structure:
+ folder/
+ ├── config.json # Metadata, Schema Info, Parameters
+ ├── data.json # Pure Knowledge Data
+ └── index/ # Vector Index Files
Args:
folder_path: Target folder path for saving.
"""
folder = Path(folder_path)
- if folder.exists() and folder.is_file():
- raise Exception("Folder path is a file, please provide a folder path.")
-
- if not folder.exists():
- folder.mkdir(parents=True, exist_ok=True)
+ if folder.is_file():
+ raise FileExistsError(f"Path is a file, expected directory: {folder}")
+ folder.mkdir(parents=True, exist_ok=True)
- try:
- # 1. Save structured data
- data = {
- "schema_name": self._data_schema.__name__,
- "data_schema": self._data_schema.model_json_schema(),
- "data": self.data.model_dump(),
- "metadata": self._prepare_metadata_for_dump(),
- }
+ # 1. Save Configuration (Schema, Metadata, Parameters)
+ self._save_config(folder)
- # Allow subclasses to add extra fields
- data.update(self._get_extra_dump_data())
+ # 2. Save Data (Pure Data)
+ self._save_data(folder)
- json_str = json.dumps(data, indent=2, ensure_ascii=False, default=str)
- data_file = folder / "state.json"
- with open(data_file, "w", encoding="utf-8") as f:
- f.write(json_str)
-
- # 2. Save vector index (via hook for flexibility)
- if self._index is not None:
- self._dump_index_storage(folder)
-
- except Exception:
- raise
+ # 3. Save Index (Optional)
+ if self._index is not None:
+ self._dump_index_storage(folder)
def load(self, folder_path: str | Path) -> None:
"""Loads knowledge from a specified folder.
- Loads from the folder:
- 1. Structured data (self._data) - loaded from state.json
- 2. Vector index (self._index) - loaded via _load_index_storage hook
-
Args:
folder_path: Source folder path containing saved knowledge.
"""
folder = Path(folder_path)
if not folder.is_dir():
- raise ValueError(f"Folder does not exist: {folder_path}")
+ raise FileNotFoundError(f"Folder not found: {folder_path}")
+
+ # 1. Load Configuration (Validate compatibility)
+ self._load_config(folder)
- # 1. Load structured data
- data_file = folder / "state.json"
- if not data_file.is_file():
- raise ValueError(f"Data file not found: {data_file}")
+ # 2. Load Data (Restore state)
+ self._load_data(folder)
- with open(data_file, "r", encoding="utf-8") as f:
- data = json.load(f)
+ # 3. Load Index (Restore vector search)
+ self._load_index_storage(folder)
- # Load data using hook for proper state sync
- loaded_data = self._data_schema.model_validate(data.get("data", {}))
- self._set_data_state(loaded_data)
+ # ==================== Serialization: Components ====================
+
+ def _save_config(self, folder: Path) -> None:
+ """Saves metadata, schema info, and instance parameters."""
+ config = {
+ "version": "1.0",
+ "type": self.__class__.__name__,
+ "schema": self._data_schema.__name__,
+ "params": {
+ "prompt": self.prompt,
+ "chunk_size": self.chunk_size,
+ "chunk_overlap": self.chunk_overlap,
+ "max_workers": self.max_workers,
+ },
+ "metadata": {
+ k: str(v) if isinstance(v, datetime) else v
+ for k, v in self.metadata.items()
+ },
+ "extra": self._get_extra_config_data(),
+ }
- # Update metadata with loaded values
- if "metadata" in data:
- self.metadata.update(data["metadata"])
- self.metadata["updated_at"] = datetime.now()
+ with open(folder / "config.json", "w", encoding="utf-8") as f:
+ json.dump(config, f, indent=2, ensure_ascii=False)
- # Allow subclasses to load extra data
- self._load_extra_data(data)
+ def _save_data(self, folder: Path) -> None:
+ """Saves the pure knowledge data."""
+ export_data = self.data.model_dump()
- # 2. Load vector index (via hook for flexibility)
- self._load_index_storage(folder)
+ with open(folder / "data.json", "w", encoding="utf-8") as f:
+ json.dump(export_data, f, indent=2, ensure_ascii=False, default=str)
- # ==================== Serialization: Index Storage Hooks ====================
+ def _load_config(self, folder: Path) -> None:
+ """Loads and validates configuration."""
+ config_path = folder / "config.json"
+ if not config_path.exists():
+ raise FileNotFoundError(f"Config file not found: {config_path}")
+ with open(config_path, "r", encoding="utf-8") as f:
+ config = json.load(f)
+
+ # Strict schema validation - raise error if mismatch
+ if config.get("schema") != self._data_schema.__name__:
+ raise ValueError(
+ f"Schema mismatch: Expected '{self._data_schema.__name__}', "
+ f"but found '{config.get('schema')}' in config.json"
+ )
+
+ # Restore metadata
+ if "metadata" in config:
+ self.metadata.update(config["metadata"])
+
+ # Restore extra config
+ if "extra" in config:
+ self._load_extra_config_data(config["extra"])
+
+ def _load_data(self, folder: Path) -> None:
+ """Loads data and restores internal state."""
+ data_path = folder / "data.json"
+ if not data_path.exists():
+ raise FileNotFoundError(f"Data file not found: {data_path}")
+
+ with open(data_path, "r", encoding="utf-8") as f:
+ raw_data = json.load(f)
+
+ # Validate and Set State
+ validated_data = self._data_schema.model_validate(raw_data)
+ self._set_data_state(validated_data)
+
+ @abstractmethod
def _dump_index_storage(self, folder: Path) -> None:
"""HOOK: Save vector index to disk.
- Default Implementation: FAISS save_local
-
- Subclasses can override to support:
- - Alternative vector store implementations (Chroma, Pinecone, etc.)
- - Custom serialization logic
- - Multiple indices
+ Subclasses must implement to support their specific vector store
+ (FAISS, Chroma, Pinecone, etc.).
Args:
folder: Target folder for saving index files.
"""
- index_path = str(folder / "faiss_index")
- self._index.save_local(index_path)
+ pass
+ @abstractmethod
def _load_index_storage(self, folder: Path) -> None:
"""HOOK: Load vector index from disk.
- Default Implementation: FAISS load_local
-
- Subclasses can override to support:
- - Alternative vector store implementations (Chroma, Pinecone, etc.)
- - Custom deserialization logic
- - Multiple indices
+ Subclasses must implement to support their specific vector store
+ (FAISS, Chroma, Pinecone, etc.).
Args:
folder: Source folder containing index files.
"""
- from langchain_community.vectorstores import FAISS
-
- index_path = str(folder / "faiss_index")
- if Path(index_path).exists():
- try:
- self._index = FAISS.load_local(
- index_path, self.embedder, allow_dangerous_deserialization=True
- )
- except Exception:
- self._index = None
- else:
- self._index = None
-
- # ==================== Serialization: Helper Methods ====================
+ pass
- def _prepare_metadata_for_dump(self) -> Dict[str, Any]:
- """Helper to serialize metadata values (e.g., datetimes)."""
- return {
- k: str(v) if isinstance(v, datetime) else v
- for k, v in self.metadata.items()
- }
+ # ==================== Serialization: Hooks ====================
- def _get_extra_dump_data(self) -> Dict[str, Any]:
- """Hook for subclasses to add extra fields to state.json.
+ def _get_extra_config_data(self) -> Dict[str, Any]:
+ """Hook to save subclass-specific configuration (not data).
- Override this method to save subclass-specific data (e.g., merge strategy).
+ Override this method to save subclass-specific config like merge strategy.
Returns:
- Dictionary of extra data to include in state.json.
+ Dictionary of extra config data.
"""
return {}
- def _load_extra_data(self, json_data: Dict[str, Any]) -> None:
- """Hook for subclasses to extract extra fields from state.json.
+ def _load_extra_config_data(self, extra_config: Dict[str, Any]) -> None:
+ """Hook to load subclass-specific configuration.
- Override this method to restore subclass-specific data.
+ Override this method to restore subclass-specific config.
Args:
- json_data: The loaded JSON data dictionary.
+ extra_config: Extra configuration data from config.json.
"""
pass
@@ -545,7 +555,7 @@ class BaseAutoType(ABC, Generic[T]):
)
# Merge the data from both instances
- merged_data = self.merge_batch([self._data, other._data])
+ merged_data = self.merge_batch_data([self._data, other._data])
# Create a new instance with the same configuration
new_instance = self._create_empty_instance()
diff --git a/hyperextract/core/list.py b/hyperextract/core/list.py
index 6c88478..a3df171 100644
--- a/hyperextract/core/list.py
+++ b/hyperextract/core/list.py
@@ -1,5 +1,6 @@
"""List Knowledge Pattern - extracts a collection of objects from text."""
+from pathlib import Path
from typing import (
List,
Any,
@@ -69,7 +70,7 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
chunk_overlap: int = 200,
max_workers: int = 10,
show_progress: bool = True,
- **kwargs,
+ fields_for_index: List[str] | None = None,
):
"""Initialize AutoList with item schema and configuration.
@@ -82,8 +83,11 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
chunk_overlap: Overlapping characters between adjacent chunks.
max_workers: Maximum concurrent extraction tasks.
show_progress: Whether to log progress information.
+ fields_for_index: Optional list of field names to include in vector index.
+ If None, all fields are indexed.
"""
self.item_schema = item_schema
+ self.fields_for_index = fields_for_index
container_name = f"{item_schema.__name__}List"
self.item_list_schema = create_model(
@@ -103,7 +107,6 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
chunk_overlap=chunk_overlap,
max_workers=max_workers,
show_progress=show_progress,
- **kwargs,
)
def _default_prompt(self) -> str:
@@ -138,6 +141,7 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
chunk_overlap=self.chunk_overlap,
max_workers=self.max_workers,
show_progress=self.show_progress,
+ fields_for_index=self.fields_for_index,
)
@property
@@ -158,10 +162,6 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
"""
self._data = self.item_list_schema()
- def _init_index_state(self) -> None:
- """Initialize vector index to empty state."""
- self._index = None
-
def _set_data_state(self, data: AutoListSchema) -> None:
"""
SET: Full reset. Replace with new data (e.g., load from disk).
@@ -177,13 +177,17 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
For AutoList, incremental update means appending new items to existing list.
"""
if incoming_data.items:
- merged_data = self.merge_batch([self._data, incoming_data])
+ merged_data = self.merge_batch_data([self._data, incoming_data])
self._data = merged_data
self.clear_index()
+ def _init_index_state(self) -> None:
+ """Initialize vector index to empty state."""
+ self._index = None
+
# ==================== Core Methods ====================
- def merge_batch(self, data_list: List[AutoListSchema]) -> AutoListSchema:
+ def merge_batch_data(self, data_list: List[AutoListSchema]) -> AutoListSchema:
"""Pure data merge method implementing list append strategy.
Merge strategy: Collects all items from all container objects and merges them
@@ -199,16 +203,17 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
"""
all_items = []
- # Collect all items from each container
for data in data_list:
- copied_data = data.model_copy(deep=True)
- all_items.extend(copied_data.items)
+ all_items.extend(data.items)
- # Return a new container object
return self.item_list_schema(items=all_items)
def build_index(self) -> None:
- """Builds independent vector index for each item in the list."""
+ """Builds independent vector index for each item in the list.
+
+ If fields_for_index is specified, only those fields are indexed.
+ Otherwise, all fields are indexed.
+ """
items = self.items
if not items:
logger.warning("No items to index")
@@ -219,8 +224,16 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
documents = []
for idx, item in enumerate(items):
- # Serialize each Item as a Document
- content = item.model_dump_json() # Use JSON string as content
+ # Extract content based on fields_for_index
+ if self.fields_for_index:
+ # Only index specified fields
+ item_dict = item.model_dump()
+ indexed_fields = {k: v for k, v in item_dict.items() if k in self.fields_for_index}
+ content = str(indexed_fields)
+ else:
+ # Index all fields
+ content = item.model_dump_json()
+
documents.append(
Document(
page_content=content,
@@ -231,7 +244,7 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
if documents:
try:
self._index = FAISS.from_documents(documents, self.embedder)
- logger.info(f"Built FAISS index with {len(documents)} items")
+ logger.info(f"Built FAISS index with {len(documents)} items (fields: {self.fields_for_index or 'all'})")
except ImportError:
logger.error("FAISS not available. Install with: pip install faiss-cpu")
raise
@@ -268,6 +281,30 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
logger.info(f"Found {len(results)} results for query: {query[:50]}...")
return results
+ # ==================== Index Storage ====================
+
+ def _dump_index_storage(self, folder: Path) -> None:
+ """Saves FAISS vector index to disk."""
+ if self._index is None:
+ return
+ index_path = str(folder / "index")
+ self._index.save_local(index_path)
+
+ def _load_index_storage(self, folder: Path) -> None:
+ """Loads FAISS vector index from disk."""
+ from langchain_community.vectorstores import FAISS
+
+ index_path = folder / "index"
+ if index_path.exists():
+ try:
+ self._index = FAISS.load_local(
+ str(index_path), self.embedder, allow_dangerous_deserialization=True
+ )
+ except Exception:
+ self._index = None
+ else:
+ self._index = None
+
# ==================== Pythonic Sequence Operations ====================
def __len__(self) -> int:
@@ -710,11 +747,6 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
self._data.items.reverse()
self.metadata["updated_at"] = datetime.now()
- # Rebuild index if it exists to maintain consistency
- if self._index is not None:
- self._index = None
- self.build_index()
-
def sort(
self, key: Union[Callable[[Item], Any], None] = None, reverse: bool = False
) -> None:
@@ -749,11 +781,6 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
self._data.items.sort(key=key, reverse=reverse)
self.metadata["updated_at"] = datetime.now()
- # Rebuild index if it exists to maintain consistency
- if self._index is not None:
- self._index = None
- self.build_index()
-
# ==================== Helper Methods ====================
def _validate_item_schema(self, item: Any) -> None:
diff --git a/hyperextract/core/model.py b/hyperextract/core/model.py
index eac53b9..17fcdf6 100644
--- a/hyperextract/core/model.py
+++ b/hyperextract/core/model.py
@@ -1,5 +1,6 @@
"""Unit Knowledge Pattern - extracts a single structured object from text."""
+from pathlib import Path
from typing import List, Any, Type
from datetime import datetime
from langchain_core.language_models.chat_models import BaseChatModel
@@ -115,13 +116,13 @@ class AutoModel(BaseAutoType[T]):
For AutoModel, incremental update means filling missing fields, first extraction wins.
"""
- merged_data = self.merge_batch([self._data, incoming_data])
+ merged_data = self.merge_batch_data([self._data, incoming_data])
self._data = merged_data
self.clear_index()
# ==================== Core Methods ====================
- def merge_batch(self, data_list: List[T]) -> T:
+ def merge_batch_data(self, data_list: List[T]) -> T:
"""Pure data merge method implementing field-level update strategy.
Merge strategy: First extraction takes precedence. Subsequent extractions only fill
@@ -211,6 +212,30 @@ class AutoModel(BaseAutoType[T]):
logger.info(f"Found {len(results)} results for query: {query[:50]}...")
return results
+ # ==================== Index Storage ====================
+
+ def _dump_index_storage(self, folder: Path) -> None:
+ """Saves FAISS vector index to disk."""
+ if self._index is None:
+ return
+ index_path = str(folder / "index")
+ self._index.save_local(index_path)
+
+ def _load_index_storage(self, folder: Path) -> None:
+ """Loads FAISS vector index from disk."""
+ from langchain_community.vectorstores import FAISS
+
+ index_path = folder / "index"
+ if index_path.exists():
+ try:
+ self._index = FAISS.load_local(
+ str(index_path), self.embedder, allow_dangerous_deserialization=True
+ )
+ except Exception:
+ self._index = None
+ else:
+ self._index = None
+
# ==================== Operators ====================
def __add__(self, other):
diff --git a/hyperextract/core/set.py b/hyperextract/core/set.py
index 9038bf4..686aec8 100644
--- a/hyperextract/core/set.py
+++ b/hyperextract/core/set.py
@@ -4,7 +4,18 @@ Provides automatic deduplication based on a user-specified unique key field.
Supports multiple merge strategies including LLM-powered intelligent merging.
"""
-from typing import List, Any, Type, TypeVar, Generic, Optional, Callable, Union
+from pathlib import Path
+from typing import (
+ List,
+ Any,
+ Type,
+ TypeVar,
+ Generic,
+ Optional,
+ Callable,
+ Union,
+ TYPE_CHECKING,
+)
from datetime import datetime
from ontomem import OMem
from ontomem.merger import MergeStrategy, create_merger, BaseMerger
@@ -15,13 +26,7 @@ from langchain_core.documents import Document
from langchain_community.vectorstores import FAISS
from .base import BaseAutoType
-
-try:
- from hyperextract.config import logger
-except ImportError:
- import logging
-
- logger = logging.getLogger(__name__)
+from hyperextract.config import logger
Item = TypeVar("Item", bound=BaseModel)
@@ -70,6 +75,10 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
1
"""
+ if TYPE_CHECKING:
+ # Use generic version during type checking to maintain complete type hints
+ item_set_schema: Type[AutoSetSchema[Item]]
+
def __init__(
self,
item_schema: Type[Item],
@@ -77,7 +86,9 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
embedder: Embeddings,
*,
key_extractor: Callable[[Item], Any],
- strategy_or_merger: Union[MergeStrategy, BaseMerger] = MergeStrategy.LLM.PREFER_INCOMING,
+ strategy_or_merger: Union[
+ MergeStrategy, BaseMerger
+ ] = MergeStrategy.LLM.PREFER_INCOMING,
prompt: str = "",
chunk_size: int = 2000,
chunk_overlap: int = 200,
@@ -130,7 +141,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
if kwargs:
logger.warning(
"Initialized with a Merger instance. Additional kwargs are ignored: %s",
- list(kwargs.keys())
+ list(kwargs.keys()),
)
self._merger = strategy_or_merger
else:
@@ -298,7 +309,9 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
# ==================== Core Override Methods ====================
- def merge_batch(self, data_list: List[AutoSetSchema[Item]]) -> AutoSetSchema[Item]:
+ def merge_batch_data(
+ self, data_list: List[AutoSetSchema[Item]]
+ ) -> AutoSetSchema[Item]:
"""Merges multiple data containers with automatic deduplication.
Pure function: Does not modify internal state.
@@ -326,13 +339,20 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
# Get strategy value for logging
if isinstance(self.strategy_or_merger, BaseMerger):
- strategy_str = self.strategy_or_merger.strategy if hasattr(self.strategy_or_merger, 'strategy') else "custom_merger"
+ strategy_str = (
+ self.strategy_or_merger.strategy
+ if hasattr(self.strategy_or_merger, "strategy")
+ else "custom_merger"
+ )
else:
- strategy_str = self.strategy_or_merger.value if hasattr(self.strategy_or_merger, 'value') else str(self.strategy_or_merger)
+ strategy_str = (
+ self.strategy_or_merger.value
+ if hasattr(self.strategy_or_merger, "value")
+ else str(self.strategy_or_merger)
+ )
logger.info(
- f"Merged into {len(merged_items)} unique items "
- f"(strategy: {strategy_str})"
+ f"Merged into {len(merged_items)} unique items (strategy: {strategy_str})"
)
return self.item_set_schema(items=merged_items)
@@ -347,13 +367,21 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
"""
# Get strategy value: handle different types of strategy_or_merger
if isinstance(self.strategy_or_merger, BaseMerger):
- strategy_value = self.strategy_or_merger.strategy if hasattr(self.strategy_or_merger, 'strategy') else "custom_merger"
+ strategy_value = (
+ self.strategy_or_merger.strategy
+ if hasattr(self.strategy_or_merger, "strategy")
+ else "custom_merger"
+ )
elif isinstance(self.strategy_or_merger, str):
strategy_value = self.strategy_or_merger
else:
# MergeStrategy enum
- strategy_value = self.strategy_or_merger.value if hasattr(self.strategy_or_merger, 'value') else str(self.strategy_or_merger)
-
+ strategy_value = (
+ self.strategy_or_merger.value
+ if hasattr(self.strategy_or_merger, "value")
+ else str(self.strategy_or_merger)
+ )
+
return {
"item_schema_name": self.item_schema.__name__,
"strategy_or_merger": strategy_value,
@@ -422,6 +450,30 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
logger.info(f"Found {len(results)} results for query: {query[:50]}...")
return results
+ # ==================== Index Storage ====================
+
+ def _dump_index_storage(self, folder: Path) -> None:
+ """Saves FAISS vector index to disk."""
+ if self._index is None:
+ return
+ index_path = str(folder / "index")
+ self._index.save_local(index_path)
+
+ def _load_index_storage(self, folder: Path) -> None:
+ """Loads FAISS vector index from disk."""
+ from langchain_community.vectorstores import FAISS
+
+ index_path = folder / "index"
+ if index_path.exists():
+ try:
+ self._index = FAISS.load_local(
+ str(index_path), self.embedder, allow_dangerous_deserialization=True
+ )
+ except Exception:
+ self._index = None
+ else:
+ self._index = None
+
# ==================== Set-Specific Methods ====================
def add(self, item: Item) -> None: