Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
重构知识管理逻辑,移除索引脏标记,添加清空索引方法以简化数据管理
yifanfeng9754a7a0bbdf7c
diff --git a/src/knowledge/base.py b/src/knowledge/base.pyindex c9e307d..5c733ef 100644--- a/src/knowledge/base.py+++ b/src/knowledge/base.py@@ -74,7 +74,6 @@ class BaseKnowledge(ABC, Generic[T]):# 向量存储相关(使用 FAISS)self._index = None- self._index_dirty: bool = True # 标记索引是否需要重建# 内部状态:存储提取的知识self._data: T = self._data_schema()@@ -114,10 +113,15 @@ class BaseKnowledge(ABC, Generic[T]):"""return self._data- @abstractmethoddef clear(self):- """清空所有知识。"""- pass+ """清空所有知识(数据和索引)。"""+ self.metadata["updated_at"] = datetime.now()+ self._data = self._data_schema()+ self.clear_index()++ def clear_index(self):+ """清空向量索引(不影响数据)。"""+ self._index = None# ==================== 提取与聚合 ====================@@ -131,7 +135,7 @@ class BaseKnowledge(ABC, Generic[T]):2. 判断文本长度,决定是否分块3. 对每个块进行提取4. 调用 merge() 合并数据(merge 不修改内部状态)- 5. extract 负责将 merge 的返回值赋值给 self._data 并更新 self._index_dirty+ 5. extract 负责将 merge 的返回值赋值给 self._data 并调用 clear_index()6. 返回提取到的结果:param text: 输入文本(可以是短文本或长文本)diff --git a/src/knowledge/entity.py b/src/knowledge/entity.pyindex 3ac86c8..10a89dc 100644--- a/src/knowledge/entity.py+++ b/src/knowledge/entity.py@@ -126,7 +126,6 @@ Please merge these two entities intelligently and return the result.""""""清空所有实体"""super().clear()self._entity_map.clear()- logger.info("Cleared all entities")# ==================== 提取与聚合 ====================@@ -216,7 +215,7 @@ Please merge these two entities intelligently and return the result."""# 将去重后的结果写回 self.itemsself._data.items = list(self._entity_map.values())- self._index_dirty = True+ self.clear_index()logger.info(f"Merged {len(self._entity_map)} unique entities "@@ -265,8 +264,8 @@ Please merge these two entities intelligently and return the result."""logger.warning("No entities to index")return- if not self._index_dirty and self._index is not None:- return # 索引已是最新+ if self._index is not None:+ return # 索引已存在documents = []for entity in self._entity_map.values():@@ -294,7 +293,6 @@ Please merge these two entities intelligently and return the result.""")self._index = FAISS.from_documents(documents, self.embedder)- self._index_dirty = Falselogger.info(f"Built FAISS index with {len(documents)} entities")@@ -311,8 +309,8 @@ Please merge these two entities intelligently and return the result."""logger.warning("No items to search")return []- if self._index is None or self._index_dirty:- raise Exception("Index is not built or dirty, please build the index first.")+ if self._index is None:+ raise Exception("Index is not built, please build the index first.")docs = self._index.similarity_search(query, k=top_k)@@ -366,7 +364,7 @@ Please merge these two entities intelligently and return the result."""if name in self._entity_map:del self._entity_map[name]self._data.items = list(self._entity_map.values())- self._index_dirty = True+ self.clear_index()logger.info(f"Removed entity: {name}")return Truereturn Falsediff --git a/src/knowledge/generic.py b/src/knowledge/generic.pyindex f56ea8b..1a7d8d4 100644--- a/src/knowledge/generic.py+++ b/src/knowledge/generic.py@@ -87,16 +87,6 @@ class UnitKnowledge(BaseKnowledge[T]):"### Source Text:\n")- # ==================== 数据管理 ====================-- def clear(self):- """清空所有知识"""- self.metadata["updated_at"] = datetime.now()- self._data = self.data_schema()- self._index = None- self._index_dirty = True- logger.info("Cleared all knowledge")-# ==================== 提取与聚合 ====================def extract(self, text: str, *, merge_mode: bool = False) -> T:@@ -147,7 +137,7 @@ class UnitKnowledge(BaseKnowledge[T]):self._data = self.merge(extracted_data_list)# 统一修改状态- self._index_dirty = True+ self.clear_index()self.metadata["updated_at"] = datetime.now()logger.info("Knowledge extraction completed")@@ -182,7 +172,7 @@ class UnitKnowledge(BaseKnowledge[T]):logger.warning("No data to index")return- if not self._index_dirty and self._index is not None:+ if self._index is not None:returndocuments = []@@ -197,7 +187,6 @@ class UnitKnowledge(BaseKnowledge[T]):)self._index = FAISS.from_documents(documents, self.embedder)- self._index_dirty = Falselogger.info(f"Built FAISS index with {len(documents)} documents")def search(self, query: str, top_k: int = 3) -> List[Any]:@@ -213,10 +202,8 @@ class UnitKnowledge(BaseKnowledge[T]):logger.warning("No items to search")return []- if self._index is None or self._index_dirty:- raise Exception(- "Index is not built or dirty, please build the index first."- )+ if self._index is None:+ raise Exception("Index is not built, please build the index first.")docs = self._index.similarity_search(query, k=top_k)@@ -325,16 +312,13 @@ class UnitKnowledge(BaseKnowledge[T]):self._index = FAISS.load_local(index_path, self.embedder, allow_dangerous_deserialization=True)- self._index_dirty = Falselogger.info(f"Loaded FAISS index from {index_path}")except Exception as e:logger.warning(f"Failed to load FAISS index: {e}")self._index = None- self._index_dirty = Trueelse:logger.warning("No index file found, will rebuild on next search")self._index = None- self._index_dirty = Truelogger.info(f"Loaded knowledge successfully with {self.size()} fields")@@ -396,7 +380,10 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):container_name = f"{item_schema.__name__}List"self.item_list_schema = create_model(container_name,- items=(List[item_schema], Field(default_factory=list, description="Item list"))+ items=(+ List[item_schema],+ Field(default_factory=list, description="Item list"),+ ),)super().__init__(@@ -427,14 +414,6 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):"""获取内部列表"""return getattr(self._data, "items", [])- def clear(self):- """清空所有列表项"""- self.metadata["updated_at"] = datetime.now()- self._data = self.item_list_schema(items=[])- self._index = None- self._index_dirty = True- logger.info("Cleared list knowledge")-def extract(self, text: str, *, merge_mode: bool = False) -> ItemListSchema:"""使用 LangChain 原生批处理提取列表@@ -483,7 +462,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):self._data = self.merge(extracted_data_list)# 统一修改状态- self._index_dirty = True+ self.clear_index()self.metadata["updated_at"] = datetime.now()logger.info("Knowledge extraction completed")@@ -519,7 +498,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):logger.warning("No items to index")return- if not self._index_dirty and self._index is not None:+ if self._index is not None:returndocuments = []@@ -535,10 +514,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):if documents:try:- from langchain_community.vectorstores import FAISS-self._index = FAISS.from_documents(documents, self.embedder)- self._index_dirty = Falselogger.info(f"Built FAISS index with {len(documents)} items")except ImportError:logger.error("FAISS not available. Install with: pip install faiss-cpu")@@ -644,22 +620,18 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):index_path = str(folder / "faiss_index")if Path(index_path).exists():try:- from langchain_community.vectorstores import FAISS-self._index = FAISS.load_local(index_path, self.embedder, allow_dangerous_deserialization=True)- self._index_dirty = Falselogger.info(f"Loaded FAISS index from {index_path}")except Exception as e:logger.warning(f"Failed to load FAISS index: {e}")self._index = None- self._index_dirty = Trueelse:logger.warning("No index file found, will rebuild on next search")self._index = None- self._index_dirty = Truelogger.info(f"Loaded knowledge successfully with {self.size()} items")+# Set Knowledge