Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

重构知识管理逻辑,移除索引脏标记,添加清空索引方法以简化数据管理

yifanfeng9754a7a0bbdf7c
diff --git a/src/knowledge/base.py b/src/knowledge/base.py
index c9e307d..5c733ef 100644
--- a/src/knowledge/base.py
+++ b/src/knowledge/base.py
@@ -74,7 +74,6 @@ class BaseKnowledge(ABC, Generic[T]):
# 向量存储相关(使用 FAISS)
self._index = None
- self._index_dirty: bool = True # 标记索引是否需要重建
# 内部状态:存储提取的知识
self._data: T = self._data_schema()
@@ -114,10 +113,15 @@ class BaseKnowledge(ABC, Generic[T]):
"""
return self._data
- @abstractmethod
def clear(self):
- """清空所有知识。"""
- pass
+ """清空所有知识(数据和索引)。"""
+ self.metadata["updated_at"] = datetime.now()
+ self._data = self._data_schema()
+ self.clear_index()
+
+ def clear_index(self):
+ """清空向量索引(不影响数据)。"""
+ self._index = None
# ==================== 提取与聚合 ====================
@@ -131,7 +135,7 @@ class BaseKnowledge(ABC, Generic[T]):
2. 判断文本长度,决定是否分块
3. 对每个块进行提取
4. 调用 merge() 合并数据(merge 不修改内部状态)
- 5. extract 负责将 merge 的返回值赋值给 self._data 并更新 self._index_dirty
+ 5. extract 负责将 merge 的返回值赋值给 self._data 并调用 clear_index()
6. 返回提取到的结果
:param text: 输入文本(可以是短文本或长文本)
diff --git a/src/knowledge/entity.py b/src/knowledge/entity.py
index 3ac86c8..10a89dc 100644
--- a/src/knowledge/entity.py
+++ b/src/knowledge/entity.py
@@ -126,7 +126,6 @@ Please merge these two entities intelligently and return the result."""
"""清空所有实体"""
super().clear()
self._entity_map.clear()
- logger.info("Cleared all entities")
# ==================== 提取与聚合 ====================
@@ -216,7 +215,7 @@ Please merge these two entities intelligently and return the result."""
# 将去重后的结果写回 self.items
self._data.items = list(self._entity_map.values())
- self._index_dirty = True
+ self.clear_index()
logger.info(
f"Merged {len(self._entity_map)} unique entities "
@@ -265,8 +264,8 @@ Please merge these two entities intelligently and return the result."""
logger.warning("No entities to index")
return
- if not self._index_dirty and self._index is not None:
- return # 索引已是最新
+ if self._index is not None:
+ return # 索引已存在
documents = []
for entity in self._entity_map.values():
@@ -294,7 +293,6 @@ Please merge these two entities intelligently and return the result."""
)
self._index = FAISS.from_documents(documents, self.embedder)
- self._index_dirty = False
logger.info(f"Built FAISS index with {len(documents)} entities")
@@ -311,8 +309,8 @@ Please merge these two entities intelligently and return the result."""
logger.warning("No items to search")
return []
- if self._index is None or self._index_dirty:
- raise Exception("Index is not built or dirty, please build the index first.")
+ if self._index is None:
+ raise Exception("Index is not built, please build the index first.")
docs = self._index.similarity_search(query, k=top_k)
@@ -366,7 +364,7 @@ Please merge these two entities intelligently and return the result."""
if name in self._entity_map:
del self._entity_map[name]
self._data.items = list(self._entity_map.values())
- self._index_dirty = True
+ self.clear_index()
logger.info(f"Removed entity: {name}")
return True
return False
diff --git a/src/knowledge/generic.py b/src/knowledge/generic.py
index f56ea8b..1a7d8d4 100644
--- a/src/knowledge/generic.py
+++ b/src/knowledge/generic.py
@@ -87,16 +87,6 @@ class UnitKnowledge(BaseKnowledge[T]):
"### Source Text:\n"
)
- # ==================== 数据管理 ====================
-
- def clear(self):
- """清空所有知识"""
- self.metadata["updated_at"] = datetime.now()
- self._data = self.data_schema()
- self._index = None
- self._index_dirty = True
- logger.info("Cleared all knowledge")
-
# ==================== 提取与聚合 ====================
def extract(self, text: str, *, merge_mode: bool = False) -> T:
@@ -147,7 +137,7 @@ class UnitKnowledge(BaseKnowledge[T]):
self._data = self.merge(extracted_data_list)
# 统一修改状态
- self._index_dirty = True
+ self.clear_index()
self.metadata["updated_at"] = datetime.now()
logger.info("Knowledge extraction completed")
@@ -182,7 +172,7 @@ class UnitKnowledge(BaseKnowledge[T]):
logger.warning("No data to index")
return
- if not self._index_dirty and self._index is not None:
+ if self._index is not None:
return
documents = []
@@ -197,7 +187,6 @@ class UnitKnowledge(BaseKnowledge[T]):
)
self._index = FAISS.from_documents(documents, self.embedder)
- self._index_dirty = False
logger.info(f"Built FAISS index with {len(documents)} documents")
def search(self, query: str, top_k: int = 3) -> List[Any]:
@@ -213,10 +202,8 @@ class UnitKnowledge(BaseKnowledge[T]):
logger.warning("No items to search")
return []
- if self._index is None or self._index_dirty:
- raise Exception(
- "Index is not built or dirty, please build the index first."
- )
+ if self._index is None:
+ raise Exception("Index is not built, please build the index first.")
docs = self._index.similarity_search(query, k=top_k)
@@ -325,16 +312,13 @@ class UnitKnowledge(BaseKnowledge[T]):
self._index = FAISS.load_local(
index_path, self.embedder, allow_dangerous_deserialization=True
)
- self._index_dirty = False
logger.info(f"Loaded FAISS index from {index_path}")
except Exception as e:
logger.warning(f"Failed to load FAISS index: {e}")
self._index = None
- self._index_dirty = True
else:
logger.warning("No index file found, will rebuild on next search")
self._index = None
- self._index_dirty = True
logger.info(f"Loaded knowledge successfully with {self.size()} fields")
@@ -396,7 +380,10 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
container_name = f"{item_schema.__name__}List"
self.item_list_schema = create_model(
container_name,
- items=(List[item_schema], Field(default_factory=list, description="Item list"))
+ items=(
+ List[item_schema],
+ Field(default_factory=list, description="Item list"),
+ ),
)
super().__init__(
@@ -427,14 +414,6 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
"""获取内部列表"""
return getattr(self._data, "items", [])
- def clear(self):
- """清空所有列表项"""
- self.metadata["updated_at"] = datetime.now()
- self._data = self.item_list_schema(items=[])
- self._index = None
- self._index_dirty = True
- logger.info("Cleared list knowledge")
-
def extract(self, text: str, *, merge_mode: bool = False) -> ItemListSchema:
"""
使用 LangChain 原生批处理提取列表
@@ -483,7 +462,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
self._data = self.merge(extracted_data_list)
# 统一修改状态
- self._index_dirty = True
+ self.clear_index()
self.metadata["updated_at"] = datetime.now()
logger.info("Knowledge extraction completed")
@@ -519,7 +498,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
logger.warning("No items to index")
return
- if not self._index_dirty and self._index is not None:
+ if self._index is not None:
return
documents = []
@@ -535,10 +514,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
if documents:
try:
- from langchain_community.vectorstores import FAISS
-
self._index = FAISS.from_documents(documents, self.embedder)
- self._index_dirty = False
logger.info(f"Built FAISS index with {len(documents)} items")
except ImportError:
logger.error("FAISS not available. Install with: pip install faiss-cpu")
@@ -644,22 +620,18 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
index_path = str(folder / "faiss_index")
if Path(index_path).exists():
try:
- from langchain_community.vectorstores import FAISS
-
self._index = FAISS.load_local(
index_path, self.embedder, allow_dangerous_deserialization=True
)
- self._index_dirty = False
logger.info(f"Loaded FAISS index from {index_path}")
except Exception as e:
logger.warning(f"Failed to load FAISS index: {e}")
self._index = None
- self._index_dirty = True
else:
logger.warning("No index file found, will rebuild on next search")
self._index = None
- self._index_dirty = True
logger.info(f"Loaded knowledge successfully with {self.size()} items")
+
# Set Knowledge