Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

重构状态管理方法,统一命名为 _init_data_state、_set_data_state 和 _update_data_state,增强代码一致性和可读性

yifanfeng9774d69bd4a8c0
diff --git a/examples/auto_set_demo.py b/examples/auto_set_demo.py
index 0f73b66..5ee460d 100644
--- a/examples/auto_set_demo.py
+++ b/examples/auto_set_demo.py
@@ -66,7 +66,7 @@ def main():
llm_client=llm,
embedder=embedder,
key_extractor=lambda x: x.name, # 基于怪物名称去重
- merge_item_strategy=MergeStrategy.LLM_MERGE,
+ strategy_or_merger="llm_prefer_incoming",
prompt="""从冒险者的日志记录中提取所有怪物的相关信息。
对于每只怪物,请提供:
diff --git a/hyperextract/core/base.py b/hyperextract/core/base.py
index 85fbcc8..e6aaf60 100644
--- a/hyperextract/core/base.py
+++ b/hyperextract/core/base.py
@@ -30,6 +30,8 @@ class BaseAutoType(ABC, Generic[T]):
- Provide serialization and deserialization capabilities
"""
+ # ==================== Initialization & Configuration ====================
+
def __init__(
self,
data_schema: Type[T],
@@ -77,11 +79,7 @@ class BaseAutoType(ABC, Generic[T]):
separators=["\n\n", "\n", "。", "!", "?", ". ", "! ", "? ", " ", ""],
)
- # Vector index using FAISS for semantic search
- self._index = None
-
# Internal state storing the extracted knowledge
- # NOTE: self._data will be initialized by _init_internal_state hook (subclass responsibility)
self.metadata: Dict[str, Any] = {
"created_at": datetime.now(),
"updated_at": datetime.now(),
@@ -112,12 +110,12 @@ class BaseAutoType(ABC, Generic[T]):
@abstractmethod
def _default_prompt(self) -> str:
"""Returns the default extraction prompt template.
-
+
Subclasses must implement this to provide a prompt tailored to their extraction pattern.
"""
pass
- # ==================== Data Management ====================
+ # ==================== Data Access Interface ====================
@property
def data_schema(self) -> Type[T]:
@@ -142,30 +140,53 @@ class BaseAutoType(ABC, Generic[T]):
"""
pass
- # ==================== State Management Lifecycle Hooks ====================
+ # ==================== Data Management Operations ====================
+
+ def clear(self):
+ """Clears all knowledge including data and vector index."""
+ self._init_internal_state()
+ self.metadata["updated_at"] = datetime.now()
+
+ def clear_index(self):
+ """Clears the vector index without affecting the stored data."""
+ self._init_index_state()
+
+ # ==================== Lifecycle Hooks: State Management ====================
- @abstractmethod
def _init_internal_state(self) -> None:
+ """Master control: Initialize or reset all internal state (INIT/RESET).
+
+ This concrete method orchestrates the reset process by calling two hooks:
+ 1. _init_data_state() - Subclass responsibility: reset data structures
+ 2. _init_index_state() - Default: reset index, can be overridden by subclass
+
+ Called in two scenarios:
+ - During __init__ to set up the initial state
+ - When clear() is called to reset to empty state
"""
- Protected hook to initialize internal state (INIT).
- Called at the END of __init__ to ensure all basic attributes are set first.
+ self._init_data_state()
+ self._init_index_state()
- Responsibilities:
- 1. Initialize self._data with appropriate structure (may differ from Schema T)
- 2. Initialize self._index = None
+ @abstractmethod
+ def _init_data_state(self) -> None:
+ """HOOK: Initialize or reset data structures to empty state.
+
+ Subclass Responsibility:
+ - Initialize self._data with appropriate structure (may differ from Schema T)
+ - Reset any auxiliary data structures (e.g., lookup dicts, caches)
Subclasses must implement this to set up internal structures that may be optimized
- beyond the standard Pydantic schema (e.g., dict-based for AutoSet, graph structures).
+ beyond the standard Pydantic schema (e.g., OMem for AutoSet, dict-based for others).
"""
pass
@abstractmethod
- def _set_internal_state(self, data: T) -> None:
- """
- Protected hook to overwrite internal state (SET).
+ def _set_data_state(self, data: T) -> None:
+ """HOOK: Overwrite data state with new data (SET).
+
Called by extract() or load() where the data provided IS the new state.
- Responsibilities:
+ Subclass Responsibilities:
1. Replace self._data with new data (full reset)
2. Convert standard Schema T to optimized internal structure if needed
3. Invalidate vector index (self.clear_index())
@@ -176,13 +197,13 @@ class BaseAutoType(ABC, Generic[T]):
pass
@abstractmethod
- def _update_internal_state(self, incoming_data: T) -> None:
- """
- Protected hook to merge new data into state (UPDATE).
+ def _update_data_state(self, incoming_data: T) -> None:
+ """HOOK: Merge new data into state (UPDATE).
+
Called by feed() where the data provided is INCREMENTAL.
- Responsibilities:
- 1. Merge incoming_data into current internal state (optimized for incremental updates)
+ Subclass Responsibilities:
+ 1. Merge incoming_data into current data state (optimized for incremental updates)
2. Invalidate vector index (self.clear_index())
Subclasses should implement optimized incremental updates (e.g., set.add instead of
@@ -194,27 +215,18 @@ class BaseAutoType(ABC, Generic[T]):
pass
@abstractmethod
- def _clear_internal_state(self) -> None:
- """
- Protected hook to fully clear internal state (CLEAR).
- Called by clear() method.
+ def _init_index_state(self) -> None:
+ """HOOK: Initialize or reset vector index to empty state.
- Responsibilities:
- 1. Reset internal state to empty (matching _init_internal_state)
+ Subclass Responsibility:
+ - Initialize or reset vector index structures
+ - Can be FAISS, Chroma, Pinecone, or custom implementation
+ - Typically sets self._index = None or initializes specific index instance
+
+ This separation allows index implementation to be decoupled from base class.
"""
pass
- # ==================== Data Management ====================
-
- def clear(self):
- """Clears all knowledge including data and vector index."""
- self._clear_internal_state()
- self.metadata["updated_at"] = datetime.now()
-
- def clear_index(self):
- """Clears the vector index without affecting the stored data."""
- self._index = None
-
# ==================== Extraction & Merge ====================
def _extract_data(self, text: str) -> T:
@@ -256,7 +268,7 @@ class BaseAutoType(ABC, Generic[T]):
extracted_data = self._extract_data(text)
new_instance = self._create_empty_instance()
- new_instance._set_internal_state(extracted_data)
+ new_instance._set_data_state(extracted_data)
new_instance.metadata["created_at"] = datetime.now()
new_instance.metadata["updated_at"] = datetime.now()
@@ -277,10 +289,10 @@ class BaseAutoType(ABC, Generic[T]):
Self (the current instance).
"""
extracted_data = self._extract_data(text)
-
+
# Use UPDATE hook instead of manual merge+set
- self._update_internal_state(extracted_data)
-
+ self._update_data_state(extracted_data)
+
self.metadata["updated_at"] = datetime.now()
return self
@@ -307,7 +319,7 @@ class BaseAutoType(ABC, Generic[T]):
"""
pass
- # ==================== Indexing & Query ====================
+ # ==================== Indexing & Search ====================
@abstractmethod
def build_index(self):
@@ -338,14 +350,14 @@ class BaseAutoType(ABC, Generic[T]):
"""
pass
- # ==================== Serialization ====================
+ # ==================== Serialization: Core Interface ====================
def dump(self, folder_path: str | Path) -> None:
"""Exports knowledge to a specified folder.
Saves to the folder:
1. Structured data (self._data) - saved as state.json
- 2. Vector index (self._index) - saved as FAISS index files
+ 2. Vector index (self._index) - saved via _dump_index_storage hook
Args:
folder_path: Target folder path for saving.
@@ -374,10 +386,9 @@ class BaseAutoType(ABC, Generic[T]):
with open(data_file, "w", encoding="utf-8") as f:
f.write(json_str)
- # 2. Save vector index
+ # 2. Save vector index (via hook for flexibility)
if self._index is not None:
- index_path = str(folder / "faiss_index")
- self._index.save_local(index_path)
+ self._dump_index_storage(folder)
except Exception:
raise
@@ -387,13 +398,11 @@ class BaseAutoType(ABC, Generic[T]):
Loads from the folder:
1. Structured data (self._data) - loaded from state.json
- 2. Vector index (self._index) - loaded from FAISS index files
+ 2. Vector index (self._index) - loaded via _load_index_storage hook
Args:
folder_path: Source folder path containing saved knowledge.
"""
- from langchain_community.vectorstores import FAISS
-
folder = Path(folder_path)
if not folder.is_dir():
raise ValueError(f"Folder does not exist: {folder_path}")
@@ -408,7 +417,7 @@ class BaseAutoType(ABC, Generic[T]):
# Load data using hook for proper state sync
loaded_data = self._data_schema.model_validate(data.get("data", {}))
- self._set_internal_state(loaded_data)
+ self._set_data_state(loaded_data)
# Update metadata with loaded values
if "metadata" in data:
@@ -418,7 +427,42 @@ class BaseAutoType(ABC, Generic[T]):
# Allow subclasses to load extra data
self._load_extra_data(data)
- # 2. Load vector index
+ # 2. Load vector index (via hook for flexibility)
+ self._load_index_storage(folder)
+
+ # ==================== Serialization: Index Storage Hooks ====================
+
+ def _dump_index_storage(self, folder: Path) -> None:
+ """HOOK: Save vector index to disk.
+
+ Default Implementation: FAISS save_local
+
+ Subclasses can override to support:
+ - Alternative vector store implementations (Chroma, Pinecone, etc.)
+ - Custom serialization logic
+ - Multiple indices
+
+ Args:
+ folder: Target folder for saving index files.
+ """
+ index_path = str(folder / "faiss_index")
+ self._index.save_local(index_path)
+
+ def _load_index_storage(self, folder: Path) -> None:
+ """HOOK: Load vector index from disk.
+
+ Default Implementation: FAISS load_local
+
+ Subclasses can override to support:
+ - Alternative vector store implementations (Chroma, Pinecone, etc.)
+ - Custom deserialization logic
+ - Multiple indices
+
+ Args:
+ folder: Source folder containing index files.
+ """
+ from langchain_community.vectorstores import FAISS
+
index_path = str(folder / "faiss_index")
if Path(index_path).exists():
try:
@@ -427,8 +471,10 @@ class BaseAutoType(ABC, Generic[T]):
)
except Exception:
self._index = None
+ else:
+ self._index = None
- # ==================== Serialization Helpers ====================
+ # ==================== Serialization: Helper Methods ====================
def _prepare_metadata_for_dump(self) -> Dict[str, Any]:
"""Helper to serialize metadata values (e.g., datetimes)."""
@@ -505,7 +551,7 @@ class BaseAutoType(ABC, Generic[T]):
new_instance = self._create_empty_instance()
# Set the merged data using hook and update metadata
- new_instance._set_internal_state(merged_data)
+ new_instance._set_data_state(merged_data)
new_instance.metadata["created_at"] = min(
self.metadata["created_at"], other.metadata["created_at"]
)
diff --git a/hyperextract/core/list.py b/hyperextract/core/list.py
index 2810d1a..6c88478 100644
--- a/hyperextract/core/list.py
+++ b/hyperextract/core/list.py
@@ -151,14 +151,18 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
# ==================== State Management Lifecycle Hooks ====================
- def _init_internal_state(self) -> None:
+ def _init_data_state(self) -> None:
"""
- INIT: Initialize with empty schema.
+ INIT/RESET: Initialize or reset with empty schema.
+ Called during __init__ and when clear() is called.
"""
self._data = self.item_list_schema()
+
+ def _init_index_state(self) -> None:
+ """Initialize vector index to empty state."""
self._index = None
- def _set_internal_state(self, data: AutoListSchema) -> None:
+ def _set_data_state(self, data: AutoListSchema) -> None:
"""
SET: Full reset. Replace with new data (e.g., load from disk).
Called by extract() or load() where data IS the new state.
@@ -166,7 +170,7 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
self._data = data
self.clear_index()
- def _update_internal_state(self, incoming_data: AutoListSchema) -> None:
+ def _update_data_state(self, incoming_data: AutoListSchema) -> None:
"""
UPDATE: Incremental merge. Append incoming items to current list (called by feed()).
@@ -177,13 +181,6 @@ class AutoList(BaseAutoType[AutoListSchema[Item]], Generic[Item]):
self._data = merged_data
self.clear_index()
- def _clear_internal_state(self) -> None:
- """
- CLEAR: Reset to empty schema instance.
- """
- self._data = self.item_list_schema()
- self.clear_index()
-
# ==================== Core Methods ====================
def merge_batch(self, data_list: List[AutoListSchema]) -> AutoListSchema:
diff --git a/hyperextract/core/model.py b/hyperextract/core/model.py
index 1549a37..eac53b9 100644
--- a/hyperextract/core/model.py
+++ b/hyperextract/core/model.py
@@ -90,14 +90,18 @@ class AutoModel(BaseAutoType[T]):
# ==================== State Management Lifecycle Hooks ====================
- def _init_internal_state(self) -> None:
+ def _init_data_state(self) -> None:
"""
- INIT: Initialize with empty schema instance.
+ INIT/RESET: Initialize or reset with empty schema instance.
+ Called during __init__ and when clear() is called.
"""
self._data = self._data_schema()
+
+ def _init_index_state(self) -> None:
+ """Initialize vector index to empty state."""
self._index = None
- def _set_internal_state(self, data: T) -> None:
+ def _set_data_state(self, data: T) -> None:
"""
SET: Full reset. Replace with new data (e.g., load from disk).
Called by extract() or load() where data IS the new state.
@@ -105,7 +109,7 @@ class AutoModel(BaseAutoType[T]):
self._data = data
self.clear_index()
- def _update_internal_state(self, incoming_data: T) -> None:
+ def _update_data_state(self, incoming_data: T) -> None:
"""
UPDATE: Incremental merge. Merge fields with field-level update strategy (called by feed()).
@@ -115,13 +119,6 @@ class AutoModel(BaseAutoType[T]):
self._data = merged_data
self.clear_index()
- def _clear_internal_state(self) -> None:
- """
- CLEAR: Reset to empty schema instance.
- """
- self._data = self._data_schema()
- self.clear_index()
-
# ==================== Core Methods ====================
def merge_batch(self, data_list: List[T]) -> T:
diff --git a/hyperextract/core/set.py b/hyperextract/core/set.py
index 3ae957c..9038bf4 100644
--- a/hyperextract/core/set.py
+++ b/hyperextract/core/set.py
@@ -4,13 +4,15 @@ Provides automatic deduplication based on a user-specified unique key field.
Supports multiple merge strategies including LLM-powered intelligent merging.
"""
-from typing import List, Any, Type, TypeVar, Generic, Optional, Callable
+from typing import List, Any, Type, TypeVar, Generic, Optional, Callable, Union
from datetime import datetime
from ontomem import OMem
-from ontomem.merger import MergeStrategy, create_merger
+from ontomem.merger import MergeStrategy, create_merger, BaseMerger
from pydantic import BaseModel, Field, create_model
from langchain_core.language_models.chat_models import BaseChatModel
from langchain_core.embeddings import Embeddings
+from langchain_core.documents import Document
+from langchain_community.vectorstores import FAISS
from .base import BaseAutoType
@@ -75,13 +77,14 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
embedder: Embeddings,
*,
key_extractor: Callable[[Item], Any],
- merge_item_strategy: MergeStrategy = MergeStrategy.LLM.PREFER_INCOMING,
+ strategy_or_merger: Union[MergeStrategy, BaseMerger] = MergeStrategy.LLM.PREFER_INCOMING,
prompt: str = "",
chunk_size: int = 2000,
chunk_overlap: int = 200,
max_workers: int = 10,
show_progress: bool = True,
llm_batch_size: int = 10,
+ **kwargs: Any,
):
"""Initialize AutoSet with key extractor and merge strategy.
@@ -90,13 +93,17 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
llm_client: Language model client for extraction and merging.
embedder: Embedding model for vector indexing.
key_extractor: Function to extract unique key from an item (required).
- merge_item_strategy: Strategy for merging duplicate items (default: KEEP_NEW).
+ strategy_or_merger: Merge strategy or pre-configured merger instance. Can be:
+ 1. A MergeStrategy enum value (e.g., MergeStrategy.LLM.BALANCED)
+ 2. A pre-configured BaseMerger instance (for full control)
prompt: Custom extraction prompt.
chunk_size: Maximum characters per chunk for long texts.
chunk_overlap: Overlapping characters between adjacent chunks.
max_workers: Maximum concurrent extraction tasks.
show_progress: Whether to log progress information.
llm_batch_size: Batch size for LLM merge operations (for LLM_MERGE strategy).
+ **kwargs: Additional arguments passed to create_merger() when strategy_or_merger is
+ a MergeStrategy enum. Ignored if strategy_or_merger is a BaseMerger instance.
"""
# Store item_schema
@@ -112,31 +119,31 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
),
)
- super().__init__(
- data_schema=self.item_set_schema,
- llm_client=llm_client,
- embedder=embedder,
- prompt=prompt or self._default_prompt(),
- chunk_size=chunk_size,
- chunk_overlap=chunk_overlap,
- max_workers=max_workers,
- show_progress=show_progress,
- )
-
- # AutoSet-specific attributes (before _init_internal_state is called by parent)
+ # AutoSet-specific attributes (MUST be initialized BEFORE super().__init__ calls _init_internal_state)
self.key_extractor = key_extractor
- self.merge_item_strategy = merge_item_strategy
+ self.strategy_or_merger = strategy_or_merger
self.llm_batch_size = llm_batch_size
- # Create Merger instance using ontomem's create_merger
- self._merger = create_merger(
- strategy=merge_item_strategy,
- key_extractor=key_extractor,
- llm_client=llm_client,
- item_schema=item_schema,
- )
+ # Setup Merge Strategy
+ if isinstance(strategy_or_merger, BaseMerger):
+ # Pre-configured merger instance: use directly
+ if kwargs:
+ logger.warning(
+ "Initialized with a Merger instance. Additional kwargs are ignored: %s",
+ list(kwargs.keys())
+ )
+ self._merger = strategy_or_merger
+ else:
+ # MergeStrategy enum: create merger with strategy and pass kwargs
+ self._merger = create_merger(
+ strategy=strategy_or_merger,
+ key_extractor=key_extractor,
+ llm_client=llm_client,
+ item_schema=item_schema,
+ **kwargs, # Pass additional arguments to create_merger
+ )
- # Initialize OMem instance for storage and deduplication
+ # Initialize OMem instance BEFORE calling super().__init__ so _init_internal_state can use it
self._data: OMem[Item] = OMem(
memory_schema=item_schema,
key_extractor=key_extractor,
@@ -146,6 +153,17 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
verbose=show_progress,
)
+ super().__init__(
+ data_schema=self.item_set_schema,
+ llm_client=llm_client,
+ embedder=embedder,
+ prompt=prompt or self._default_prompt(),
+ chunk_size=chunk_size,
+ chunk_overlap=chunk_overlap,
+ max_workers=max_workers,
+ show_progress=show_progress,
+ )
+
# ==================== Override Instance Creation ====================
def _create_empty_instance(self) -> "AutoSet[Item]":
@@ -161,7 +179,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
llm_client=self.llm_client,
embedder=self.embedder,
key_extractor=self.key_extractor,
- merge_item_strategy=self.merge_item_strategy,
+ strategy_or_merger=self.strategy_or_merger,
prompt=self.prompt,
chunk_size=self.chunk_size,
chunk_overlap=self.chunk_overlap,
@@ -209,7 +227,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
def __len__(self) -> int:
"""Returns the number of unique items in the set."""
- return len(self._data)
+ return len(self._data.items)
def __contains__(self, key: Any) -> bool:
"""Checks if a unique key exists in the set.
@@ -245,14 +263,18 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
# ==================== State Management Lifecycle Overrides ====================
- def _init_internal_state(self) -> None:
+ def _init_data_state(self) -> None:
"""
- INIT: Initialize OMem as empty.
+ INIT/RESET: Initialize or reset OMem as empty.
+ Called during __init__ and when clear() is called.
"""
self._data.clear()
- self.clear_index()
- def _set_internal_state(self, data: AutoSetSchema[Item]) -> None:
+ def _init_index_state(self) -> None:
+ """Initialize vector index to empty state."""
+ self._index = None
+
+ def _set_data_state(self, data: AutoSetSchema[Item]) -> None:
"""
SET: Full Reset. Wipe OMem and refill from data (e.g., load from disk).
Called by extract() or load() where data IS the new state.
@@ -262,7 +284,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
self._data.add(data.items)
self.clear_index()
- def _update_internal_state(self, incoming_data: AutoSetSchema[Item]) -> None:
+ def _update_data_state(self, incoming_data: AutoSetSchema[Item]) -> None:
"""
UPDATE: Incremental merge. Add to OMem efficiently (called by feed()).
@@ -274,13 +296,6 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
self._data.add(incoming_data.items)
self.clear_index()
- def _clear_internal_state(self) -> None:
- """
- CLEAR: Wipe OMem (called by clear()).
- """
- self._data.clear()
- self.clear_index()
-
# ==================== Core Override Methods ====================
def merge_batch(self, data_list: List[AutoSetSchema[Item]]) -> AutoSetSchema[Item]:
@@ -309,9 +324,15 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
merged_items = self._merger.merge(all_items)
+ # Get strategy value for logging
+ if isinstance(self.strategy_or_merger, BaseMerger):
+ strategy_str = self.strategy_or_merger.strategy if hasattr(self.strategy_or_merger, 'strategy') else "custom_merger"
+ else:
+ strategy_str = self.strategy_or_merger.value if hasattr(self.strategy_or_merger, 'value') else str(self.strategy_or_merger)
+
logger.info(
f"Merged into {len(merged_items)} unique items "
- f"(strategy: {self.merge_item_strategy.value})"
+ f"(strategy: {strategy_str})"
)
return self.item_set_schema(items=merged_items)
@@ -324,11 +345,83 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
Returns:
Dictionary with extra AutoSet-specific fields.
"""
+ # Get strategy value: handle different types of strategy_or_merger
+ if isinstance(self.strategy_or_merger, BaseMerger):
+ strategy_value = self.strategy_or_merger.strategy if hasattr(self.strategy_or_merger, 'strategy') else "custom_merger"
+ elif isinstance(self.strategy_or_merger, str):
+ strategy_value = self.strategy_or_merger
+ else:
+ # MergeStrategy enum
+ strategy_value = self.strategy_or_merger.value if hasattr(self.strategy_or_merger, 'value') else str(self.strategy_or_merger)
+
return {
"item_schema_name": self.item_schema.__name__,
- "merge_item_strategy": self.merge_item_strategy.value,
+ "strategy_or_merger": strategy_value,
}
+ # ==================== Indexing & Query ====================
+
+ def build_index(self) -> None:
+ """Builds independent vector index for each item in the set."""
+ items = self.items
+ if not items:
+ logger.warning("No items to index")
+ return
+
+ if self._index is not None:
+ return
+
+ documents = []
+ for idx, item in enumerate(items):
+ # Serialize each Item as a Document
+ content = item.model_dump_json() # Use JSON string as content
+ documents.append(
+ Document(
+ page_content=content,
+ metadata={"raw": item.model_dump(), "index": idx},
+ )
+ )
+
+ if documents:
+ try:
+ self._index = FAISS.from_documents(documents, self.embedder)
+ logger.info(f"Built FAISS index with {len(documents)} items")
+ except ImportError:
+ logger.error("FAISS not available. Install with: pip install faiss-cpu")
+ raise
+
+ def search(self, query: str, top_k: int = 3) -> List[Item]:
+ """Searches items in the set using semantic similarity.
+
+ Args:
+ query: Search query string.
+ top_k: Number of results to return.
+
+ Returns:
+ List of relevant items.
+ """
+ if not self.items:
+ logger.warning("No items to search")
+ return []
+
+ if self._index is None:
+ raise Exception("Vector store not initialized")
+
+ docs = self._index.similarity_search(query, k=top_k)
+ results: List[Item] = []
+ for doc in docs:
+ # Attempt to restore as object
+ try:
+ raw = doc.metadata.get("raw", {})
+ item = self.item_schema.model_validate(raw)
+ results.append(item)
+ except Exception as e:
+ logger.warning(f"Failed to restore item: {e}")
+ results.append(doc.metadata.get("raw"))
+
+ logger.info(f"Found {len(results)} results for query: {query[:50]}...")
+ return results
+
# ==================== Set-Specific Methods ====================
def add(self, item: Item) -> None: