Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
重构索引构建和搜索功能,优化文档处理和错误日志记录
yifanfeng9742e7eacee379
diff --git a/hyperextract/knowledge/generic/set.py b/hyperextract/knowledge/generic/set.pyindex d99fa0a..b381b6f 100644--- a/hyperextract/knowledge/generic/set.py+++ b/hyperextract/knowledge/generic/set.py@@ -402,69 +402,6 @@ Please merge these two items intelligently and return the merged result."""# Return items listreturn self.items if store else merged_data.items- def build_index(self):- """Builds independent vector index for each unique item."""- items = self.items- if not items:- logger.warning("No items to index")- return-- if self._index is not None:- return-- documents = []- for idx, item in enumerate(items):- # Serialize each Item as a Document- content = item.model_dump_json() # Use JSON string as content- documents.append(- Document(- page_content=content,- metadata={"raw": item.model_dump(), "index": idx},- )- )-- if documents:- try:- from langchain_community.vectorstores import FAISS-- self._index = FAISS.from_documents(documents, self.embedder)- logger.info(f"Built FAISS index with {len(documents)} items")- except ImportError:- logger.error("FAISS not available. Install with: pip install faiss-cpu")- raise-- def search(self, query: str, top_k: int = 3) -> List[Any]:- """Searches items in the set using semantic similarity.-- Args:- query: Search query string.- top_k: Number of results to return.-- Returns:- List of relevant items.- """- if not self.items:- logger.warning("No items to search")- return []-- if self._index is None:- raise Exception("Vector store not initialized")-- docs = self._index.similarity_search(query, k=top_k)- results = []- for doc in docs:- # Attempt to restore as object- try:- raw = doc.metadata.get("raw", {})- item = self.item_schema.model_validate(raw)- results.append(item)- except Exception as e:- logger.warning(f"Failed to restore item: {e}")- results.append(doc.metadata.get("raw"))-- logger.info(f"Found {len(results)} results for query: {query[:50]}...")- return results-def merge(self, data_list: List[ItemSetSchema[Item]]) -> ItemSetSchema[Item]:"""Merges multiple data containers with automatic deduplication.@@ -676,6 +613,69 @@ Please merge these two items intelligently and return the merged result."""logger.error(f"LLM merge failed: {e}")return incoming # Fallback to incoming item+ def build_index(self):+ """Builds independent vector index for each unique item."""+ items = self.items+ if not items:+ logger.warning("No items to index")+ return++ if self._index is not None:+ return++ documents = []+ for idx, item in enumerate(items):+ # Serialize each Item as a Document+ content = item.model_dump_json() # Use JSON string as content+ documents.append(+ Document(+ page_content=content,+ metadata={"raw": item.model_dump(), "index": idx},+ )+ )++ if documents:+ try:+ from langchain_community.vectorstores import FAISS++ self._index = FAISS.from_documents(documents, self.embedder)+ logger.info(f"Built FAISS index with {len(documents)} items")+ except ImportError:+ logger.error("FAISS not available. Install with: pip install faiss-cpu")+ raise++ def search(self, query: str, top_k: int = 3) -> List[Any]:+ """Searches items in the set using semantic similarity.++ Args:+ query: Search query string.+ top_k: Number of results to return.++ Returns:+ List of relevant items.+ """+ if not self.items:+ logger.warning("No items to search")+ return []++ if self._index is None:+ raise Exception("Vector store not initialized")++ docs = self._index.similarity_search(query, k=top_k)+ results = []+ for doc in docs:+ # Attempt to restore as object+ try:+ raw = doc.metadata.get("raw", {})+ item = self.item_schema.model_validate(raw)+ results.append(item)+ except Exception as e:+ logger.warning(f"Failed to restore item: {e}")+ results.append(doc.metadata.get("raw"))++ logger.info(f"Found {len(results)} results for query: {query[:50]}...")+ return results+# ==================== Serialization ====================def dump(self, folder_path: str | Path):