Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
Refactor entity and knowledge extraction modules for improved clarity and consistency
yifanfeng9744134716cae5
diff --git a/examples/generic_knowledge_demo.py b/examples/generic_knowledge_demo.pyindex 90d0155..7da8bb0 100644--- a/examples/generic_knowledge_demo.py+++ b/examples/generic_knowledge_demo.py@@ -85,7 +85,7 @@ def main():# 6. 构建索引print("\n[5] 构建向量索引...")knowledge.build_index()- print(f"✓ 索引构建完成,共 {knowledge.size()} 个字段")+ print(f"✓ 索引构建完成,共 {len(knowledge)} 个字段")# 7. 语义搜索print("\n[6] 执行语义搜索...")@@ -112,7 +112,7 @@ def main():new_knowledge.load(save_path)print("✓ 知识加载完成")print(f" 加载的标题: {new_knowledge.data.title}")- print(f" 加载的字段数: {new_knowledge.size()}")+ print(f" 加载的字段数: {len(new_knowledge)}")# 10. 测试长文本提取(分块)print("\n[9] 测试长文本提取(自动分块)...")diff --git a/examples/list_knowledge_multi_chunk_demo.py b/examples/list_knowledge_multi_chunk_demo.pyindex 3e66df8..2880108 100644--- a/examples/list_knowledge_multi_chunk_demo.py+++ b/examples/list_knowledge_multi_chunk_demo.py@@ -132,7 +132,7 @@ def main():print("\n[1] 生成测试新闻文本...")news_text = generate_long_news_text()print(f" 文本长度: {len(news_text)} 字符")- print(f" 预计分块数: {len(news_text) // 200 + 1} 块 (chunk_size=200)")+ print(f" 预计分块数: {len(news_text) // 300 + 1} 块 (chunk_size=300)")# 2. 初始化 LLM 和 Embedderprint("\n[2] 初始化 LLM 和 Embedder...")@@ -140,7 +140,7 @@ def main():embedder = OpenAIEmbeddings(model="text-embedding-3-small")# 3. 创建 ListKnowledge 实例 (使用小的 chunk_size 测试多 chunk)- print("\n[3] 创建 ListKnowledge 实例 (chunk_size=200)...")+ print("\n[3] 创建 ListKnowledge 实例 (chunk_size=300)...")knowledge = ListKnowledge(item_schema=NewsEvent,llm_client=llm,@@ -173,7 +173,7 @@ def main():print(f"\n ... 还有 {len(events) - 5} 个事件未显示")# 6. 测试列表大小- print(f"\n[6] 知识列表大小: {knowledge.size()} 个事件")+ print(f"\n[6] 知识列表大小: {len(knowledge)} 个事件")# 7. 构建索引print("\n[7] 构建向量索引...")@@ -204,9 +204,9 @@ def main():"""print(" 添加补充新闻...")- before_count = knowledge.size()+ before_count = len(knowledge)knowledge.extract(additional_news, merge_mode=True)- after_count = knowledge.size()+ after_count = len(knowledge)print(f" 合并前: {before_count} 个事件")print(f" 合并后: {after_count} 个事件")print(f" 新增: {after_count - before_count} 个事件")@@ -231,14 +231,14 @@ def main():embedder=embedder,)new_knowledge.load(str(save_path))- print(f" 加载成功! 事件数: {new_knowledge.size()}")+ print(f" 加载成功! 事件数: {len(new_knowledge)}")print(f" 第一个事件: {new_knowledge.items[0].title}")# 13. 测试清空功能print("\n[12] 测试清空功能...")- print(f" 清空前: {new_knowledge.size()} 个事件")+ print(f" 清空前: {len(new_knowledge)} 个事件")new_knowledge.clear()- print(f" 清空后: {new_knowledge.size()} 个事件")+ print(f" 清空后: {len(new_knowledge)} 个事件")print("\n" + "=" * 80)print("示例运行完成!")diff --git a/examples/unit_knowledge_multi_chunk_demo.py b/examples/unit_knowledge_multi_chunk_demo.pyindex 848e961..0ccba2b 100644--- a/examples/unit_knowledge_multi_chunk_demo.py+++ b/examples/unit_knowledge_multi_chunk_demo.py@@ -181,7 +181,7 @@ def main():print(f" 关键词数量: {len(extracted_data.keywords)}")# 6. 测试字段数量- print(f"\n[6] 知识字段数: {knowledge.size()}")+ print(f"\n[6] 知识字段数: {len(knowledge)}")# 7. 构建索引print("\n[7] 构建向量索引...")@@ -228,7 +228,7 @@ def main():embedder=embedder,)new_knowledge.load(str(save_path))- print(f" 加载成功! 字段数: {new_knowledge.size()}")+ print(f" 加载成功! 字段数: {len(new_knowledge)}")print(f" 标题: {new_knowledge.data.title}")print("\n" + "=" * 80)diff --git a/src/knowledge/base.py b/src/knowledge/base.pyindex 5c733ef..c6adf12 100644--- a/src/knowledge/base.py+++ b/src/knowledge/base.py@@ -12,19 +12,22 @@ from langchain_core.prompts import ChatPromptTemplateT = TypeVar("T", bound=BaseModel)-# ===================== 知识类 (Knowledge) =====================+# ===================== Knowledge Base Class =====================class BaseKnowledge(ABC, Generic[T]):- """- 统一的知识基类 - 集提取、存储、聚合、演化于一体。-- 职责:- 1. 从文本中提取结构化知识(extract)- 2. 自动处理长文本分块- 3. 存储和聚合提取的知识- 4. 演化优化内部知识- 5. 序列化与反序列化+ """Unified knowledge base class integrating extraction, storage, aggregation, and evolution.++ This abstract base class provides a complete framework for managing structured knowledge+ extracted from text. It handles the full lifecycle from extraction to serialization.++ Responsibilities:+ - Extract structured knowledge from text using LLM+ - Automatically handle long text chunking and parallel processing+ - Store and aggregate extracted knowledge with configurable merge strategies+ - Build and maintain vector indices for semantic search+ - Support knowledge evolution and optimization+ - Provide serialization and deserialization capabilities"""def __init__(@@ -39,17 +42,17 @@ class BaseKnowledge(ABC, Generic[T]):max_workers: int = 10,show_progress: bool = True,):- """- 初始化知识对象。-- :param data_schema: 数据 Schema 类(必须是 Pydantic BaseModel 子类)- :param llm_client: LLM 客户端(用于提取和演化)- :param embedder: 向量化器(用于语义搜索和相似度计算)- :param prompt: 用户自定义的提示词(用于 extract)- :param chunk_size: 长文本分块大小- :param chunk_overlap: 分块重叠大小- :param max_workers: 最大并发数(用于并发提取)- :param show_progress: 是否显示进度信息+ """Initialize the knowledge object with schema and processing configuration.++ Args:+ data_schema: Pydantic BaseModel subclass defining the knowledge structure.+ llm_client: Language model client for extraction and evolution.+ embedder: Embedding model for semantic search and similarity computation.+ prompt: Custom prompt template for extraction (defaults to generic prompt).+ chunk_size: Maximum chunk size for splitting long texts.+ chunk_overlap: Number of overlapping characters between chunks.+ max_workers: Maximum number of concurrent extraction tasks.+ show_progress: Whether to display progress information during processing."""self._data_schema = data_schemaself.llm_client = llm_client@@ -60,22 +63,22 @@ class BaseKnowledge(ABC, Generic[T]):self.max_workers = max_workersself.show_progress = show_progress- # 初始化提示词模板和 LLM 链+ # Initialize prompt template and LLM chain for structured extractionself.prompt_template = ChatPromptTemplate.from_template(f"{self.prompt}{{chunk_text}}")self.llm_with_schema = self.llm_client.with_structured_output(self._data_schema)self.llm_chain_extract = self.prompt_template | self.llm_with_schema- # 初始化文本分割器+ # Initialize text splitter for chunking long documentsself.text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size,chunk_overlap=chunk_overlap,separators=["\n\n", "\n", "。", "!", "?", ". ", "! ", "? ", " ", ""],)- # 向量存储相关(使用 FAISS)+ # Vector index using FAISS for semantic searchself._index = None- # 内部状态:存储提取的知识+ # Internal state storing the extracted knowledgeself._data: T = self._data_schema()self.metadata: Dict[str, Any] = {"created_at": datetime.now(),@@ -84,7 +87,7 @@ class BaseKnowledge(ABC, Generic[T]):@staticmethoddef _default_prompt() -> str:- """默认提取提示词"""+ """Returns the default extraction prompt template."""return ("You are an expert knowledge extraction assistant. ""Your task is to carefully analyze the following text and extract structured information "@@ -93,157 +96,173 @@ class BaseKnowledge(ABC, Generic[T]):"### Source Text:\n")- # ==================== 数据管理 ====================+ # ==================== Data Management ====================@propertydef data_schema(self) -> Type[T]:- """- 返回当前知识类使用的 Schema 类。+ """Returns the Pydantic schema class used by this knowledge instance.- :return: Pydantic Schema 类+ Returns:+ The Pydantic BaseModel subclass defining the knowledge structure."""return self._data_schema@propertydef data(self) -> T:- """- 获取所有存储的知识(只读)。+ """Returns all stored knowledge (read-only access).- :return: 内部知识数据+ Returns:+ The internal knowledge data as a Pydantic model instance."""return self._datadef clear(self):- """清空所有知识(数据和索引)。"""+ """Clears all knowledge including data and vector index."""self.metadata["updated_at"] = datetime.now()self._data = self._data_schema()self.clear_index()def clear_index(self):- """清空向量索引(不影响数据)。"""+ """Clears the vector index without affecting the stored data."""self._index = None- # ==================== 提取与聚合 ====================+ # ==================== Extraction & Aggregation ====================@abstractmethoddef extract(self, text: str, *, merge_mode: bool = False) -> T:- """- 主提取方法 - 自动处理长文本分块、提取、聚合。-- 流程:- 1. 根据 merge_mode 决定是否清空现有数据- 2. 判断文本长度,决定是否分块- 3. 对每个块进行提取- 4. 调用 merge() 合并数据(merge 不修改内部状态)- 5. extract 负责将 merge 的返回值赋值给 self._data 并调用 clear_index()- 6. 返回提取到的结果-- :param text: 输入文本(可以是短文本或长文本)- :param merge_mode: 合并模式(默认 False)- - False(默认): 替换模式 - 仅合并新提取的数据- - True: 累积模式 - 将现有知识与新数据合并- :return: 提取到的知识数据+ """Extracts structured knowledge from text with automatic chunking and aggregation.++ This is the main entry point for knowledge extraction. It handles the complete pipeline+ from text preprocessing to knowledge aggregation.++ Extraction pipeline:+ 1. Decide whether to clear existing data based on merge_mode+ 2. Determine if text chunking is needed based on length+ 3. Extract knowledge from each chunk using LLM+ 4. Merge all extracted data using the merge() method+ 5. Update internal state (_data) and invalidate index via clear_index()+ 6. Return the aggregated knowledge++ Args:+ text: Input text to extract knowledge from (can be short or long).+ merge_mode: Controls how new data is combined with existing knowledge.+ - False (default): Replace mode - only use newly extracted data+ - True: Accumulative mode - merge new data with existing knowledge++ Returns:+ The extracted and aggregated knowledge data."""pass@abstractmethoddef merge(self, data_list: List[T]) -> T:- """- 纯数据合并方法 - 不修改内部状态。+ """Merges multiple knowledge data objects into a single unified object.++ This is a pure data transformation method that does not modify internal state.+ Subclasses implement specific merge strategies (deduplication, conflict resolution, etc.).++ Responsibilities:+ - Implement concrete merge algorithms (deduplication, conflict resolution, etc.)+ - Return a new merged data object+ - Never modify instance attributes- 职责:- - 实现具体的合并算法(去重、冲突解决等)- - 返回合并后的新数据对象- - 不修改任何实例属性+ Args:+ data_list: List of knowledge data objects to merge.- :param data_list: 需要合并的数据列表- :return: 合并后的知识数据(新对象)+ Returns:+ A new merged knowledge object."""pass- # ==================== 演化 ====================+ # ==================== Evolution ====================def evolve(self, **kwargs) -> T:- """- 演化内部知识。- 子类必须实现此方法。+ """Evolves and optimizes the internal knowledge through inference and refinement.++ Subclasses can implement this method to perform knowledge-specific evolution strategies.- 职责:- - 推理隐含关系- - 剪枝低置信度节点- - 聚类优化- - 知识补全+ Evolution capabilities:+ - Infer implicit relationships between entities+ - Prune low-confidence nodes and connections+ - Apply clustering for better organization+ - Complete missing knowledge through reasoning- :return: 新的内部知识数据+ Returns:+ The evolved knowledge data."""pass- # ==================== 索引与查询 ====================+ # ==================== Indexing & Query ====================@abstractmethoddef build_index(self):- """- 构建或重建向量索引。+ """Builds or rebuilds the vector index for semantic search.- 子类必须实现此方法,定义如何构建向量索引。- 使用 FAISS 作为向量存储后端。+ Subclasses must implement this method to define how the vector index is constructed+ from the knowledge data. Uses FAISS as the vector store backend."""pass@abstractmethoddef search(self, query: str, top_k: int = 3) -> List[Any]:- """- 语义搜索知识。+ """Performs semantic search over the knowledge base.++ Standard search workflow:+ 1. Ensure index is built (call build_index if needed)+ 2. Use vector_store.similarity_search for retrieval+ 3. Restore original data structures from Document.metadata- 默认实现:- 1. 确保索引已构建(调用 build_index)- 2. 使用 vector_store.similarity_search 进行检索- 3. 从 Document.metadata 恢复原始数据结构+ Subclasses can override this method to implement custom search logic.- 子类可覆盖此方法实现自定义搜索逻辑。+ Args:+ query: Search query string.+ top_k: Number of results to return.- :param query: 查询字符串- :param top_k: 返回结果数- :return: 相关知识列表+ Returns:+ List of relevant knowledge items."""pass@abstractmethod- def size(self) -> int:- """- 返回知识点数量。- 子类必须实现此方法。+ def __len__(self) -> int:+ """Returns the number of knowledge items in the collection.++ Subclasses must implement this method to report their specific size metric.+ This enables using len() function on knowledge instances.- :return: 知识点数量+ Returns:+ The count of knowledge items."""pass- # ==================== 序列化 ====================+ # ==================== Serialization ====================@abstractmethoddef dump(self, folder_path: str | Path):- """- 导出知识到指定文件夹。- 子类必须实现此方法。+ """Exports knowledge to a specified folder.++ Subclasses must implement this method to persist their knowledge data.- 需要保存到文件夹内:- 1. 结构化数据 (self._data) - 保存为 JSON 文件- 2. 向量索引 (self._index) - FAISS 索引文件+ Files to save:+ 1. Structured data (self._data) - saved as JSON file+ 2. Vector index (self._index) - saved as FAISS index files- :param folder_path: 目标文件夹路径+ Args:+ folder_path: Target folder path for saving knowledge."""pass@abstractmethoddef load(self, folder_path: str | Path):- """- 从文件夹加载知识。- 子类必须实现此方法。+ """Loads knowledge from a specified folder.++ Subclasses must implement this method to restore previously saved knowledge.- 需要从文件夹加载:- 1. 结构化数据 (self._data) - 从 JSON 文件加载- 2. 向量索引 (self._index) - 从 FAISS 索引文件加载+ Files to load:+ 1. Structured data (self._data) - loaded from JSON file+ 2. Vector index (self._index) - loaded from FAISS index files- :param folder_path: 文件夹路径+ Args:+ folder_path: Source folder path containing saved knowledge."""passdiff --git a/src/knowledge/entity.py b/src/knowledge/entity.pyindex 10a89dc..e692764 100644--- a/src/knowledge/entity.py+++ b/src/knowledge/entity.py@@ -1,10 +1,8 @@-"""-实体知识提取模块。+"""Entity knowledge extraction module.-基于 ListKnowledge 的实体提取实现。+Provides entity extraction implementation based on ListKnowledge."""-from pathlib import Pathfrom typing import List, Dict, Type, TypeVar, Genericfrom pydantic import BaseModel, Field as PydanticFieldfrom langchain_core.language_models.chat_models import BaseChatModel@@ -23,36 +21,34 @@ except ImportError:logger = logging.getLogger(__name__)-# 实体类型变量+# Entity type variableE = TypeVar("E", bound="BaseEntitySchema")class BaseEntitySchema(BaseModel):- """- 实体基类 - 所有实体必须包含的核心字段。+ """Base entity class defining required core fields for all entities.- 用户可继承此类添加自定义字段。+ Users can inherit from this class to add custom fields."""- name: str = PydanticField(description="实体唯一名称", min_length=1)- description: str = PydanticField(description="实体描述")+ name: str = PydanticField(description="Unique entity name", min_length=1)+ description: str = PydanticField(description="Entity description")class Config:- """允许用户扩展字段"""+ """Configuration allowing users to extend with additional fields."""- extra = "allow" # 允许额外字段- validate_assignment = True # 赋值时验证+ extra = "allow" # Allow additional fields+ validate_assignment = True # Validate on assignmentclass EntityKnowledge(ListKnowledge[E], Generic[E]):- """- 专门用于实体提取和管理的知识类。+ """Specialized knowledge class for entity extraction and management.- 特点:- 1. 自动去重(基于 entity.name)- 2. LLM 智能合并(同名实体的字段合并)- 3. 每个实体作为独立文档建索引- 4. 支持实体级别的搜索+ Key features:+ 1. Automatic deduplication based on entity.name+ 2. Intelligent LLM-powered merging for entities with the same name+ 3. Each entity indexed as an independent document+ 4. Support for entity-level semantic search"""def __init__(@@ -64,11 +60,13 @@ class EntityKnowledge(ListKnowledge[E], Generic[E]):prompt: str = "",**kwargs,):- """- :param entity_schema: 用户自定义的实体类(必须继承 BaseEntity)- :param llm_client: LLM 客户端- :param embedder: 向量化器- :param prompt: 自定义提示词+ """Initialize EntityKnowledge with entity schema and configuration.++ Args:+ entity_schema: User-defined entity class (must inherit from BaseEntity).+ llm_client: Language model client for extraction and merging.+ embedder: Embedding model for vector indexing.+ prompt: Custom extraction prompt."""self._entity_schema = entity_schema@@ -82,12 +80,12 @@ class EntityKnowledge(ListKnowledge[E], Generic[E]):self._merge_chain = self._create_merge_chain()- # 全局实体映射(主存储,类型安全)+ # Global entity mapping (primary storage, type-safe)self._entity_map: Dict[str, E] = {}@staticmethoddef _default_prompt() -> str:- """默认实体提取提示词"""+ """Returns the default entity extraction prompt."""return ("You are an expert entity extraction assistant. ""Carefully read the following text and extract all entities "@@ -97,7 +95,7 @@ class EntityKnowledge(ListKnowledge[E], Generic[E]):)def _create_merge_chain(self):- """创建 LLM 合并 chain,用于智能合并同名实体"""+ """Creates an LLM chain for intelligently merging entities with the same name."""merge_template = """You are an expert at merging entity information.Given two instances of the same entity, intelligently merge their fields into one.@@ -120,86 +118,89 @@ Please merge these two entities intelligently and return the result."""self._entity_schema)- # ==================== 数据管理 ====================+ # ==================== Data Management ====================def clear(self):- """清空所有实体"""+ """Clears all entities from storage."""super().clear()self._entity_map.clear()- # ==================== 提取与聚合 ====================+ # ==================== Extraction & Aggregation ====================def extract(self, text: str, *, merge_mode: bool = False) -> BaseModel:- """- 提取实体知识(支持替换/累积两种模式)。+ """Extracts entity knowledge with support for replace/accumulate modes.- 模式说明:- - merge_mode=False(默认): 替换模式- 仅使用新提取的实体+ Mode explanation:+ - merge_mode=False (default): Replace mode+ Only uses newly extracted entities- - merge_mode=True: 累积模式- 保留现有实体,提取新实体并智能合并(LLM 去重)+ - merge_mode=True: Accumulate mode+ Keeps existing entities, extracts new entities and intelligently merges+ (LLM-powered deduplication)- :param text: 输入文本- :param merge_mode: 合并模式(默认 False)- :return: 提取后的实体列表容器+ Args:+ text: Input text to extract entities from.+ merge_mode: Merge mode (default False).++ Returns:+ Container with extracted entities."""- # 调用父类提取方法,传递 merge_mode+ # Call parent extraction method with merge_moderesult = super().extract(text, merge_mode=merge_mode)- # 提取完成后同步到 entity_map+ # Synchronize extracted items to entity_map after extractionself._sync_items_to_map()logger.info(- f"Entity extraction completed ({'累积' if merge_mode else '替换'} mode)"+ f"Entity extraction completed ({'Accumulate' if merge_mode else 'Replace'} mode)")logger.info(f"Total entities: {len(self._entity_map)}")return resultdef merge(self, data_list: List[BaseModel]) -> BaseModel:- """- 实体级别的智能合并(使用 LLM)。+ """Entity-level intelligent merging using LLM.- 策略:- 1. 先调用父类 merge 获得所有提取的 items- 2. 将结果赋值给 self._data- 3. 同步到 entity_map 进行 LLM 去重- 4. 返回去重后的数据+ Strategy:+ 1. Call parent merge to get all extracted items+ 2. Assign result to self._data+ 3. Synchronize to entity_map for LLM-powered deduplication+ 4. Return deduplicated data- :param data_list: 多个容器对象- :return: 合并后的实体数据+ Args:+ data_list: Multiple container objects.++ Returns:+ Merged entity data."""- # 调用父类 merge,获得合并后的结果+ # Call parent merge to get merged resultmerged_result = super().merge(data_list)- # 临时赋值给 self._data 以便 _sync_items_to_map 能访问 self.items+ # Temporarily assign to self._data so _sync_items_to_map can access self.itemsself._data = merged_result- # 同步到 entity_map 进行去重+ # Synchronize to entity_map for deduplicationself._sync_items_to_map()- # 返回最终的 self._data(已经被 _sync_items_to_map 更新)+ # Return final self._data (updated by _sync_items_to_map)return self._datadef _sync_items_to_map(self):- """- 将 self.items 同步到 entity_map,执行去重和 LLM 合并。- """+ """Synchronizes self.items to entity_map, performing deduplication and LLM merging."""total_entities = 0duplicate_count = 0- # 遍历父类的 items,与已有的 entity_map 合并+ # Traverse parent's items and merge with existing entity_mapfor entity in self.items:total_entities += 1entity_name = entity.name.strip()- if not entity_name: # 跳过空名称+ if not entity_name: # Skip empty nameslogger.warning(f"Skipping entity with empty name: {entity}")continueif entity_name in self._entity_map:- # 实体已存在 - 使用 LLM 执行合并+ # Entity exists - merge using LLMduplicate_count += 1existing = self._entity_map[entity_name]merged = self._merge_entity_with_llm(existing, entity)@@ -210,10 +211,10 @@ Please merge these two entities intelligently and return the result."""f"LLM merge failed for entity '{entity_name}', keeping existing")else:- # 新实体 - 深拷贝后添加+ # New entity - deep copy and addself._entity_map[entity_name] = entity.model_copy(deep=True)- # 将去重后的结果写回 self.items+ # Write deduplicated results back to self.itemsself._data.items = list(self._entity_map.values())self.clear_index()@@ -223,19 +224,21 @@ Please merge these two entities intelligently and return the result.""")def _merge_entity_with_llm(self, entity_a: E, entity_b: E) -> E | None:- """- 使用 LLM 智能合并两个同名实体。+ """Uses LLM to intelligently merge two entities with the same name.- :param entity_a: 第一个实体- :param entity_b: 第二个实体- :return: 合并后的实体(失败返回 None)+ Args:+ entity_a: First entity.+ entity_b: Second entity.++ Returns:+ Merged entity (returns None on failure)."""try:- # 准备输入+ # Prepare inputsentity_a_json = entity_a.model_dump_json(indent=2)entity_b_json = entity_b.model_dump_json(indent=2)- # 调用 LLM 合并+ # Call LLM for mergingmerged_entity = self._merge_chain.invoke({"entity_a": entity_a_json,@@ -250,32 +253,31 @@ Please merge these two entities intelligently and return the result."""logger.error(f"Error merging entity '{entity_a.name}' with LLM: {e}")return None- # ==================== 查询与索引 ====================+ # ==================== Indexing & Query ====================def build_index(self):- """- 为每个实体构建独立的向量索引文档。+ """Builds independent vector index document for each entity.- 每个实体 → 一个 Document:- - page_content: "{name}: {description} + 其他字段"- - metadata: {"entity": entity.model_dump()}+ Each entity → one Document:+ - page_content: "{name}: {description} + other fields"+ - metadata: {"entity": entity.model_dump()}"""if len(self._entity_map) == 0:logger.warning("No entities to index")returnif self._index is not None:- return # 索引已存在+ return # Index already existsdocuments = []for entity in self._entity_map.values():- # 构建文档内容(用于向量化)+ # Build document content for vectorizationcontent_parts = [f"name: {entity.name}",f"description: {entity.description}",]- # 添加其他字段(如果用户扩展了)+ # Add other fields if user extended the schemafor field_name in self._entity_schema.model_fields:if field_name not in ("name", "description"):value = getattr(entity, field_name, None)@@ -284,7 +286,7 @@ Please merge these two entities intelligently and return the result."""content = "\n".join(content_parts)- # 保存完整实体数据到 metadata+ # Save complete entity data to metadatadocuments.append(Document(page_content=content,@@ -297,15 +299,17 @@ Please merge these two entities intelligently and return the result."""def search(self, query: str, top_k: int = 3, return_raw: bool = False) -> List[E]:- """- 语义搜索实体。+ """Performs semantic search over entities.++ Args:+ query: Search query string.+ top_k: Number of results to return.+ return_raw: Whether to return raw entity objects (default returns dict).- :param query: 查询字符串- :param top_k: 返回结果数- :param return_raw: 是否返回原始实体对象(默认返回字典)- :return: 实体列表+ Returns:+ List of entities."""- if self.size() == 0:+ if len(self) == 0:logger.warning("No items to search")return []@@ -314,17 +318,17 @@ Please merge these two entities intelligently and return the result."""docs = self._index.similarity_search(query, k=top_k)- # 恢复实体对象+ # Restore entity objectsresults = []for doc in docs:try:entity_data = doc.metadata["raw"]if return_raw:- # 返回 Pydantic 对象+ # Return Pydantic objectentity = self._entity_schema.model_validate(entity_data)results.append(entity)else:- # 返回字典+ # Return dictionaryresults.append(entity_data)except Exception as e:logger.warning(f"Failed to restore entity: {e}")@@ -332,35 +336,35 @@ Please merge these two entities intelligently and return the result."""logger.info(f"Found {len(results)} entities for query: {query[:50]}...")return results- def size(self) -> int:- """返回实体数量(直接从 entity_map 读取)"""+ def __len__(self) -> int:+ """Returns the number of entities (reads directly from entity_map)."""return len(self._entity_map)- # ==================== 便捷属性 ====================+ # ==================== Convenience Properties ====================@propertydef entities(self) -> List[E]:- """快速访问实体列表"""+ """Quick access to entity list."""return self.items@propertydef entity_schema(self) -> Type[E]:- """返回用户定义的实体 Schema"""+ """Returns the user-defined entity schema."""return self._entity_schema@propertydef entity_names(self) -> List[str]:- """返回所有实体名称(直接从 entity_map 读取)"""+ """Returns all entity names (reads directly from entity_map)."""return list(self._entity_map.keys())- # ==================== 可选增强方法 ====================+ # ==================== Optional Enhancement Methods ====================def get_entity_by_name(self, name: str) -> E | None:- """按名称获取实体(O(1) 字典查找)"""+ """Retrieves entity by name (O(1) dictionary lookup)."""return self._entity_map.get(name)def remove_entity(self, name: str) -> bool:- """删除指定实体(从 entity_map 和 items 删除)"""+ """Removes specified entity (from both entity_map and items)."""if name in self._entity_map:del self._entity_map[name]self._data.items = list(self._entity_map.values())@@ -369,13 +373,13 @@ Please merge these two entities intelligently and return the result."""return Truereturn False- # ==================== 序列化 ====================+ # ==================== Serialization ====================def load(self, folder_path: str, **kwargs):- """加载后重建 entity_map"""+ """Loads knowledge and rebuilds entity_map."""super().load(folder_path, **kwargs)- # 从 items 重建 entity_map+ # Rebuild entity_map from itemsself._entity_map = {entity.name: entity.model_copy(deep=True) for entity in self.items}diff --git a/src/knowledge/generic.py b/src/knowledge/generic.pyindex 1a7d8d4..6f50533 100644--- a/src/knowledge/generic.py+++ b/src/knowledge/generic.py@@ -1,9 +1,10 @@-"""-基础知识模式实现。+"""Fundamental knowledge pattern implementations.-包含两种核心模式:-- UnitKnowledge: 单元知识模式(适用于摘要、元数据等)-- ListKnowledge: 列表知识模式(适用于实体、事件等列表提取)+This module provides two core knowledge patterns:+ - UnitKnowledge: Single-object pattern for extracting one structured object per document+ (suitable for summaries, metadata, etc.)+ - ListKnowledge: Collection pattern for extracting multiple items from text+ (suitable for entities, events, references, etc.)"""import json@@ -25,20 +26,24 @@ except ImportError:logger = logging.getLogger(__name__)-# ==================== 两种基础知识模式 ====================+# ==================== Two Fundamental Knowledge Patterns ====================-# UnitKnowledge: 单元知识模式+# UnitKnowledge: Single-object knowledge patternclass UnitKnowledge(BaseKnowledge[T]):- """- 单元知识模式 (Unit Knowledge Pattern) - 适用于任何 Pydantic Schema。-- 特点:- - 针对整篇文档提取 **唯一** 的一个结构化对象- - 合并策略:字段级更新 (Upsert/Patch)- - 索引策略:对对象的每个非空字段建立索引- - 使用 LangChain 原生批处理(batch_as_completed)+ """Unit Knowledge Pattern - extracts a single structured object from text.++ This pattern is designed for extracting exactly one structured object per document,+ regardless of document length. Suitable for document-level information like summaries,+ metadata, or aggregate statistics.++ Key characteristics:+ - Extraction target: One unique structured object per document+ - Merge strategy: Field-level update (Upsert/Patch) - first extraction wins,+ subsequent extractions only fill missing fields+ - Indexing strategy: Each non-null field of the object is indexed independently+ - Processing: Uses LangChain native batch processing for efficient multi-chunk handling"""def __init__(@@ -54,15 +59,17 @@ class UnitKnowledge(BaseKnowledge[T]):show_progress: bool = True,**kwargs,):- """- :param data_schema: 数据 Schema 类(必须是 Pydantic BaseModel 子类)- :param llm_client: LLM 客户端- :param embedder: 向量化器- :param prompt: 自定义提示词- :param chunk_size: 长文本分块大小- :param chunk_overlap: 分块重叠大小- :param max_workers: 最大并发数- :param show_progress: 是否显示进度+ """Initialize UnitKnowledge with schema and configuration.++ Args:+ data_schema: Pydantic BaseModel subclass defining the object structure.+ llm_client: Language model client for extraction.+ embedder: Embedding model for vector indexing.+ prompt: Custom extraction prompt (defaults to generic prompt).+ chunk_size: Maximum characters per chunk for long texts.+ chunk_overlap: Overlapping characters between adjacent chunks.+ max_workers: Maximum concurrent extraction tasks.+ show_progress: Whether to log progress information."""super().__init__(data_schema,@@ -78,7 +85,7 @@ class UnitKnowledge(BaseKnowledge[T]):@staticmethoddef _default_prompt() -> str:- """默认提取提示词"""+ """Returns the default extraction prompt for single-object extraction."""return ("You are an expert knowledge extraction assistant. ""Your task is to carefully analyze the following text and extract structured information "@@ -87,27 +94,32 @@ class UnitKnowledge(BaseKnowledge[T]):"### Source Text:\n")- # ==================== 提取与聚合 ====================+ # ==================== Extraction & Aggregation ====================def extract(self, text: str, *, merge_mode: bool = False) -> T:- """- 使用 LangChain 原生批处理提取知识+ """Extracts knowledge using LangChain native batch processing.++ Automatically handles text chunking for long documents and aggregates results+ using the field-level merge strategy.- :param text: 输入文本- :param merge_mode: 合并模式(默认 False)- - False: 替换模式 - 清空后仅使用新提取的数据- - True: 累积模式 - 将现有数据与新数据合并- :return: 提取到的知识数据+ Args:+ text: Input text to extract knowledge from.+ merge_mode: Controls data combination behavior.+ - False (default): Replace mode - clears existing data before extraction+ - True: Accumulative mode - merges new data with existing knowledge++ Returns:+ The extracted knowledge object."""start_time = datetime.now()- # 替换模式:先清空+ # In replace mode, clear existing data firstif not merge_mode:self.clear()- # 判断是否需要分块+ # Determine if chunking is needed based on text lengthif len(text) <= self.chunk_size:- # 短文本:直接提取+ # Short text: direct extraction without chunkingif self.show_progress:logger.info(f"Processing single text (length: {len(text)})...")@@ -115,7 +127,7 @@ class UnitKnowledge(BaseKnowledge[T]):extracted_data_list = [extracted_data]else:- # 长文本:分块提取+ # Long text: extract by chunkingchunks = self.text_splitter.split_text(text)logger.info(f"Split text into {len(chunks)} chunks")@@ -136,7 +148,7 @@ class UnitKnowledge(BaseKnowledge[T]):self._data = self.merge(extracted_data_list)- # 统一修改状态+ # Update state uniformlyself.clear_index()self.metadata["updated_at"] = datetime.now()@@ -147,14 +159,16 @@ class UnitKnowledge(BaseKnowledge[T]):return self.datadef merge(self, data_list: List[T]) -> T:- """- 纯数据合并方法 - 字段级更新策略。+ """Pure data merge method implementing field-level update strategy.++ Merge strategy: First extraction takes precedence. Subsequent extractions only fill+ missing fields without overwriting existing values. Implemented using model_copy(update=...).- 合并策略:第一次提取的结果为准,后续只能补充缺失字段。- 使用 model_copy(update=...) 实现第一次优先合并。+ Args:+ data_list: List of extracted data objects to merge.- :param data_list: 需要合并的数据列表- :return: 合并后的知识数据(新对象)+ Returns:+ A new merged knowledge object."""result = data_list[0].model_copy()@@ -163,12 +177,12 @@ class UnitKnowledge(BaseKnowledge[T]):return result- # ==================== 查询与索引 ====================+ # ==================== Indexing & Query ====================def build_index(self):- """从 _data 的所有字段构建索引"""- # 检查是否有数据- if self.size() == 0:+ """Builds vector index from all non-null fields in the data object."""+ # Check if there's data to index+ if len(self) == 0:logger.warning("No data to index")return@@ -190,15 +204,17 @@ class UnitKnowledge(BaseKnowledge[T]):logger.info(f"Built FAISS index with {len(documents)} documents")def search(self, query: str, top_k: int = 3) -> List[Any]:- """- 搜索所有列表字段。+ """Searches all indexed fields using semantic similarity.++ Args:+ query: Search query string.+ top_k: Number of results to return.- :param query: 查询字符串- :param top_k: 返回结果数- :return: 相关知识列表(单个 item,不是容器)+ Returns:+ List of relevant knowledge items (field-value dictionaries)."""- # 检查是否有数据- if self.size() == 0:+ # Check if there's data to search+ if len(self) == 0:logger.warning("No items to search")return []@@ -207,7 +223,7 @@ class UnitKnowledge(BaseKnowledge[T]):docs = self._index.similarity_search(query, k=top_k)- # 恢复原始对象+ # Restore original objects from metadataresults = []for doc in docs:try:@@ -219,21 +235,21 @@ class UnitKnowledge(BaseKnowledge[T]):logger.info(f"Found {len(results)} results for query: {query[:50]}...")return results- def size(self) -> int:- """返回字段数"""- return len(self.data_schema.model_fields)+ def __len__(self) -> int:+ """Returns 1 since UnitKnowledge represents a single knowledge unit."""+ return 1- # ==================== 序列化 ====================+ # ==================== Serialization ====================def dump(self, folder_path: str | Path):- """- 导出知识到指定文件夹。+ """Exports knowledge to a specified folder.- 保存到文件夹内:- 1. 结构化数据 (self._data) - 保存为 state.json- 2. 向量索引 (self._index) - FAISS 索引文件+ Saves to the folder:+ 1. Structured data (self._data) - saved as state.json+ 2. Vector index (self._index) - saved as FAISS index files- :param folder_path: 目标文件夹路径+ Args:+ folder_path: Target folder path for saving."""folder = Path(folder_path)@@ -245,7 +261,7 @@ class UnitKnowledge(BaseKnowledge[T]):folder.mkdir(parents=True, exist_ok=True)try:- # 1. 保存结构化数据+ # 1. Save structured datadata = {"schema_name": self.data_schema.__name__,"data_schema": self.data_schema.model_json_schema(),@@ -262,7 +278,7 @@ class UnitKnowledge(BaseKnowledge[T]):f.write(json_str)logger.info(f"Saved data to {data_file}")- # 2. 保存向量索引+ # 2. Save vector indexif self._index is not None:index_path = str(folder / "faiss_index")self._index.save_local(index_path)@@ -274,20 +290,20 @@ class UnitKnowledge(BaseKnowledge[T]):logger.error(f"Failed to dump knowledge: {e}")def load(self, folder_path: str | Path):- """- 从文件夹加载知识。+ """Loads knowledge from a specified folder.- 从文件夹加载:- 1. 结构化数据 (self._data) - 从 state.json 加载- 2. 向量索引 (self._index) - 从 FAISS 索引文件加载+ Loads from the folder:+ 1. Structured data (self._data) - loaded from state.json+ 2. Vector index (self._index) - loaded from FAISS index files- :param folder_path: 文件夹路径+ Args:+ folder_path: Source folder path containing saved knowledge."""folder = Path(folder_path)if not folder.is_dir():raise ValueError(f"Folder does not exist: {folder_path}")- # 1. 加载结构化数据+ # 1. Load structured datadata_file = folder / "state.json"if not data_file.is_file():raise ValueError(f"Data file not found: {data_file}")@@ -298,12 +314,12 @@ class UnitKnowledge(BaseKnowledge[T]):self._data = self.data_schema.model_validate(data.get("data", {}))- # 更新元数据+ # Update metadata with loaded valuesif "metadata" in data:self.metadata.update(data["metadata"])self.metadata["updated_at"] = datetime.now()- # 2. 加载向量索引+ # 2. Load vector indexindex_path = str(folder / "faiss_index")if Path(index_path).exists():try:@@ -320,36 +336,38 @@ class UnitKnowledge(BaseKnowledge[T]):logger.warning("No index file found, will rebuild on next search")self._index = None- logger.info(f"Loaded knowledge successfully with {self.size()} fields")+ logger.info(f"Loaded knowledge successfully with {len(self)} unit(s)")-# ==================== 列表知识模式 ===================+# ==================== List Knowledge Pattern ===================Item = TypeVar("Item", bound=BaseModel)class ItemListSchema(BaseModel, Generic[Item]):- """列表容器的通用 Schema"""+ """Generic schema container for list-based knowledge patterns."""items: List[Item] = Field(default_factory=list, description="Item list")class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):- """- 列表知识模式 (List Collection Pattern)。+ """List Knowledge Pattern - extracts a collection of objects from text.++ This pattern extracts multiple independent objects from a document, suitable for+ extracting entities, events, references, or any collection of structured items.- 特点:- - 针对整篇文档提取 **一组** 对象列表(如实体、事件、引用等)- - 合并策略:追加 (Append) + 基础去重(可由子类扩展)- - 索引策略:对列表中的每个 Item 独立建立索引+ Key characteristics:+ - Extraction target: A collection of structured objects+ - Merge strategy: Append with basic deduplication (extensible by subclasses)+ - Indexing strategy: Each item in the list is indexed independently- 与 UnitKnowledge 的区别:- - UnitKnowledge: 提取单个结构化对象(如摘要、元数据)- - ListKnowledge: 提取多个独立对象的列表(如实体列表、事件列表)+ Comparison with UnitKnowledge:+ - UnitKnowledge: Extracts a single structured object (e.g., summary, metadata)+ - ListKnowledge: Extracts multiple independent objects (e.g., entity list, event list)"""if TYPE_CHECKING:- # 类型检查时使用泛型版本,保持完整的类型提示+ # Use generic version during type checking to maintain complete type hintsitem_list_schema: Type[ItemListSchema[Item]]def __init__(@@ -365,15 +383,17 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):show_progress: bool = True,**kwargs,):- """- :param item_schema: 列表中单个元素的 Schema 类- :param llm_client: LLM 客户端- :param embedder: 向量化器- :param prompt: 自定义提示词- :param chunk_size: 长文本分块大小- :param chunk_overlap: 分块重叠大小- :param max_workers: 最大并发数- :param show_progress: 是否显示进度+ """Initialize ListKnowledge with item schema and configuration.++ Args:+ item_schema: Pydantic BaseModel subclass for individual list items.+ llm_client: Language model client for extraction.+ embedder: Embedding model for vector indexing.+ prompt: Custom extraction prompt (defaults to list-oriented prompt).+ chunk_size: Maximum characters per chunk for long texts.+ chunk_overlap: Overlapping characters between adjacent chunks.+ max_workers: Maximum concurrent extraction tasks.+ show_progress: Whether to log progress information."""self.item_schema = item_schema@@ -400,7 +420,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):@staticmethoddef _default_prompt() -> str:- """默认列表提取提示词"""+ """Returns the default extraction prompt for list-based extraction."""return ("You are an expert knowledge extraction assistant. ""Extract all relevant items from the text into a list. "@@ -411,28 +431,30 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):@propertydef items(self) -> List[Item]:- """获取内部列表"""+ """Returns the internal list of extracted items."""return getattr(self._data, "items", [])def extract(self, text: str, *, merge_mode: bool = False) -> ItemListSchema:- """- 使用 LangChain 原生批处理提取列表+ """Extracts a list of items using LangChain native batch processing.++ Args:+ text: Input text to extract items from.+ merge_mode: Controls data combination behavior.+ - False (default): Replace mode - clears existing data before extraction+ - True: Accumulative mode - merges new items with existing collection- :param text: 输入文本- :param merge_mode: 合并模式(默认 False)- - False: 替换模式 - 清空后仅使用新提取的数据- - True: 累积模式 - 将现有数据与新数据合并- :return: 提取到的 items 列表+ Returns:+ The list of extracted items."""start_time = datetime.now()- # 替换模式:先清空+ # Replace mode: clear firstif not merge_mode:self.clear()- # 判断是否需要分块+ # Determine if chunking is neededif len(text) <= self.chunk_size:- # 短文本:一次性提取+ # Short text: extract in one passif self.show_progress:logger.info(f"Processing single text (length: {len(text)})...")@@ -440,7 +462,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):extracted_data_list = [extracted_data]else:- # 长文本:分块提取+ # Long text: extract by chunkingchunks = self.text_splitter.split_text(text)logger.info(f"Split text into {len(chunks)} chunks")@@ -461,7 +483,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):self._data = self.merge(extracted_data_list)- # 统一修改状态+ # Update state uniformlyself.clear_index()self.metadata["updated_at"] = datetime.now()@@ -472,27 +494,30 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):return self.itemsdef merge(self, data_list: List[ItemListSchema]) -> ItemListSchema:- """- 纯数据合并方法 - 列表追加策略。+ """Pure data merge method implementing list append strategy.++ Merge strategy: Collects all items from all container objects and merges them+ into a single list. Subclasses can override this method to implement more+ sophisticated deduplication logic (e.g., EntityKnowledge).- 合并策略:收集所有容器对象中的 items,合并到一个大列表中。- 子类可以重写此方法实现更复杂的去重逻辑(如 EntityKnowledge)。+ Args:+ data_list: List of container objects to merge.- :param data_list: 需要合并的容器对象列表- :return: 合并后的知识数据(新的 ItemListSchema 对象)+ Returns:+ A new merged ItemListSchema object."""all_items = []- # 收集所有 items+ # Collect all items from each containerfor data in data_list:copied_data = data.model_copy(deep=True)all_items.extend(copied_data.items)- # 返回新的对象+ # Return a new container objectreturn self.item_list_schema(items=all_items)def build_index(self):- """为列表中的每个 Item 构建独立索引"""+ """Builds independent vector index for each item in the list."""items = self.itemsif not items:logger.warning("No items to index")@@ -503,8 +528,8 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):documents = []for idx, item in enumerate(items):- # 序列化每个 Item 为 Document- content = item.model_dump_json() # 使用 JSON 字符串作为内容+ # Serialize each Item as a Document+ content = item.model_dump_json() # Use JSON string as contentdocuments.append(Document(page_content=content,@@ -521,12 +546,14 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):raisedef search(self, query: str, top_k: int = 3) -> List[Any]:- """- 搜索列表中的 items。+ """Searches items in the list using semantic similarity.- :param query: 查询字符串- :param top_k: 返回结果数- :return: 相关的 item 列表+ Args:+ query: Search query string.+ top_k: Number of results to return.++ Returns:+ List of relevant items."""if not self.items:logger.warning("No items to search")@@ -538,7 +565,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):docs = self._index.similarity_search(query, k=top_k)results = []for doc in docs:- # 尝试还原为对象+ # Attempt to restore as objecttry:raw = doc.metadata.get("raw", {})item = self.item_schema.model_validate(raw)@@ -550,21 +577,22 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):logger.info(f"Found {len(results)} results for query: {query[:50]}...")return results- def size(self) -> int:- """返回列表中的元素数量"""+ def __len__(self) -> int:+ """Returns the number of elements in the list."""return len(self.items)def dump(self, folder_path: str) -> Any:- """- 导出知识到指定文件夹(复用 UnitKnowledge 的实现)。- 容器本质也是 BaseModel,可以直接使用相同的序列化逻辑。+ """Exports knowledge to a specified folder.++ Uses the same serialization logic as UnitKnowledge since the container+ is also a BaseModel."""from pathlib import Pathfolder = Path(folder_path)folder.mkdir(parents=True, exist_ok=True)- # 1. 保存结构化数据+ # 1. Save structured datadata = {"schema_name": self.item_list_schema.__name__,"item_schema_name": self.item_schema.__name__,@@ -582,7 +610,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):f.write(json_str)logger.info(f"Saved data to {data_file}")- # 2. 保存向量索引+ # 2. Save vector indexif self._index is not None:index_path = str(folder / "faiss_index")self._index.save_local(index_path)@@ -593,14 +621,14 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):return json_strdef load(self, folder_path: str, **kwargs):- """从文件夹加载知识"""+ """Loads knowledge from a specified folder."""from pathlib import Pathfolder = Path(folder_path)if not folder.is_dir():raise ValueError(f"Folder does not exist: {folder_path}")- # 1. 加载结构化数据+ # 1. Load structured datadata_file = folder / "state.json"if not data_file.is_file():raise ValueError(f"Data file not found: {data_file}")@@ -611,12 +639,12 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):self._data = self.item_list_schema.model_validate(data.get("data", {}))- # 更新元数据+ # Update metadata with loaded valuesif "metadata" in data:self.metadata.update(data["metadata"])self.metadata["updated_at"] = datetime.now()- # 2. 加载向量索引+ # 2. Load vector indexindex_path = str(folder / "faiss_index")if Path(index_path).exists():try:@@ -631,7 +659,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):logger.warning("No index file found, will rebuild on next search")self._index = None- logger.info(f"Loaded knowledge successfully with {self.size()} items")+ logger.info(f"Loaded knowledge successfully with {len(self)} items")# Set Knowledge