Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

Refactor entity and knowledge extraction modules for improved clarity and consistency

yifanfeng9744134716cae5
diff --git a/examples/generic_knowledge_demo.py b/examples/generic_knowledge_demo.py
index 90d0155..7da8bb0 100644
--- a/examples/generic_knowledge_demo.py
+++ b/examples/generic_knowledge_demo.py
@@ -85,7 +85,7 @@ def main():
# 6. 构建索引
print("\n[5] 构建向量索引...")
knowledge.build_index()
- print(f"✓ 索引构建完成,共 {knowledge.size()} 个字段")
+ print(f"✓ 索引构建完成,共 {len(knowledge)} 个字段")
# 7. 语义搜索
print("\n[6] 执行语义搜索...")
@@ -112,7 +112,7 @@ def main():
new_knowledge.load(save_path)
print("✓ 知识加载完成")
print(f" 加载的标题: {new_knowledge.data.title}")
- print(f" 加载的字段数: {new_knowledge.size()}")
+ print(f" 加载的字段数: {len(new_knowledge)}")
# 10. 测试长文本提取(分块)
print("\n[9] 测试长文本提取(自动分块)...")
diff --git a/examples/list_knowledge_multi_chunk_demo.py b/examples/list_knowledge_multi_chunk_demo.py
index 3e66df8..2880108 100644
--- a/examples/list_knowledge_multi_chunk_demo.py
+++ b/examples/list_knowledge_multi_chunk_demo.py
@@ -132,7 +132,7 @@ def main():
print("\n[1] 生成测试新闻文本...")
news_text = generate_long_news_text()
print(f" 文本长度: {len(news_text)} 字符")
- print(f" 预计分块数: {len(news_text) // 200 + 1} 块 (chunk_size=200)")
+ print(f" 预计分块数: {len(news_text) // 300 + 1} 块 (chunk_size=300)")
# 2. 初始化 LLM 和 Embedder
print("\n[2] 初始化 LLM 和 Embedder...")
@@ -140,7 +140,7 @@ def main():
embedder = OpenAIEmbeddings(model="text-embedding-3-small")
# 3. 创建 ListKnowledge 实例 (使用小的 chunk_size 测试多 chunk)
- print("\n[3] 创建 ListKnowledge 实例 (chunk_size=200)...")
+ print("\n[3] 创建 ListKnowledge 实例 (chunk_size=300)...")
knowledge = ListKnowledge(
item_schema=NewsEvent,
llm_client=llm,
@@ -173,7 +173,7 @@ def main():
print(f"\n ... 还有 {len(events) - 5} 个事件未显示")
# 6. 测试列表大小
- print(f"\n[6] 知识列表大小: {knowledge.size()} 个事件")
+ print(f"\n[6] 知识列表大小: {len(knowledge)} 个事件")
# 7. 构建索引
print("\n[7] 构建向量索引...")
@@ -204,9 +204,9 @@ def main():
"""
print(" 添加补充新闻...")
- before_count = knowledge.size()
+ before_count = len(knowledge)
knowledge.extract(additional_news, merge_mode=True)
- after_count = knowledge.size()
+ after_count = len(knowledge)
print(f" 合并前: {before_count} 个事件")
print(f" 合并后: {after_count} 个事件")
print(f" 新增: {after_count - before_count} 个事件")
@@ -231,14 +231,14 @@ def main():
embedder=embedder,
)
new_knowledge.load(str(save_path))
- print(f" 加载成功! 事件数: {new_knowledge.size()}")
+ print(f" 加载成功! 事件数: {len(new_knowledge)}")
print(f" 第一个事件: {new_knowledge.items[0].title}")
# 13. 测试清空功能
print("\n[12] 测试清空功能...")
- print(f" 清空前: {new_knowledge.size()} 个事件")
+ print(f" 清空前: {len(new_knowledge)} 个事件")
new_knowledge.clear()
- print(f" 清空后: {new_knowledge.size()} 个事件")
+ print(f" 清空后: {len(new_knowledge)} 个事件")
print("\n" + "=" * 80)
print("示例运行完成!")
diff --git a/examples/unit_knowledge_multi_chunk_demo.py b/examples/unit_knowledge_multi_chunk_demo.py
index 848e961..0ccba2b 100644
--- a/examples/unit_knowledge_multi_chunk_demo.py
+++ b/examples/unit_knowledge_multi_chunk_demo.py
@@ -181,7 +181,7 @@ def main():
print(f" 关键词数量: {len(extracted_data.keywords)}")
# 6. 测试字段数量
- print(f"\n[6] 知识字段数: {knowledge.size()}")
+ print(f"\n[6] 知识字段数: {len(knowledge)}")
# 7. 构建索引
print("\n[7] 构建向量索引...")
@@ -228,7 +228,7 @@ def main():
embedder=embedder,
)
new_knowledge.load(str(save_path))
- print(f" 加载成功! 字段数: {new_knowledge.size()}")
+ print(f" 加载成功! 字段数: {len(new_knowledge)}")
print(f" 标题: {new_knowledge.data.title}")
print("\n" + "=" * 80)
diff --git a/src/knowledge/base.py b/src/knowledge/base.py
index 5c733ef..c6adf12 100644
--- a/src/knowledge/base.py
+++ b/src/knowledge/base.py
@@ -12,19 +12,22 @@ from langchain_core.prompts import ChatPromptTemplate
T = TypeVar("T", bound=BaseModel)
-# ===================== 知识类 (Knowledge) =====================
+# ===================== Knowledge Base Class =====================
class BaseKnowledge(ABC, Generic[T]):
- """
- 统一的知识基类 - 集提取、存储、聚合、演化于一体。
-
- 职责:
- 1. 从文本中提取结构化知识(extract)
- 2. 自动处理长文本分块
- 3. 存储和聚合提取的知识
- 4. 演化优化内部知识
- 5. 序列化与反序列化
+ """Unified knowledge base class integrating extraction, storage, aggregation, and evolution.
+
+ This abstract base class provides a complete framework for managing structured knowledge
+ extracted from text. It handles the full lifecycle from extraction to serialization.
+
+ Responsibilities:
+ - Extract structured knowledge from text using LLM
+ - Automatically handle long text chunking and parallel processing
+ - Store and aggregate extracted knowledge with configurable merge strategies
+ - Build and maintain vector indices for semantic search
+ - Support knowledge evolution and optimization
+ - Provide serialization and deserialization capabilities
"""
def __init__(
@@ -39,17 +42,17 @@ class BaseKnowledge(ABC, Generic[T]):
max_workers: int = 10,
show_progress: bool = True,
):
- """
- 初始化知识对象。
-
- :param data_schema: 数据 Schema 类(必须是 Pydantic BaseModel 子类)
- :param llm_client: LLM 客户端(用于提取和演化)
- :param embedder: 向量化器(用于语义搜索和相似度计算)
- :param prompt: 用户自定义的提示词(用于 extract)
- :param chunk_size: 长文本分块大小
- :param chunk_overlap: 分块重叠大小
- :param max_workers: 最大并发数(用于并发提取)
- :param show_progress: 是否显示进度信息
+ """Initialize the knowledge object with schema and processing configuration.
+
+ Args:
+ data_schema: Pydantic BaseModel subclass defining the knowledge structure.
+ llm_client: Language model client for extraction and evolution.
+ embedder: Embedding model for semantic search and similarity computation.
+ prompt: Custom prompt template for extraction (defaults to generic prompt).
+ chunk_size: Maximum chunk size for splitting long texts.
+ chunk_overlap: Number of overlapping characters between chunks.
+ max_workers: Maximum number of concurrent extraction tasks.
+ show_progress: Whether to display progress information during processing.
"""
self._data_schema = data_schema
self.llm_client = llm_client
@@ -60,22 +63,22 @@ class BaseKnowledge(ABC, Generic[T]):
self.max_workers = max_workers
self.show_progress = show_progress
- # 初始化提示词模板和 LLM 链
+ # Initialize prompt template and LLM chain for structured extraction
self.prompt_template = ChatPromptTemplate.from_template(f"{self.prompt}{{chunk_text}}")
self.llm_with_schema = self.llm_client.with_structured_output(self._data_schema)
self.llm_chain_extract = self.prompt_template | self.llm_with_schema
- # 初始化文本分割器
+ # Initialize text splitter for chunking long documents
self.text_splitter = RecursiveCharacterTextSplitter(
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
separators=["\n\n", "\n", "。", "!", "?", ". ", "! ", "? ", " ", ""],
)
- # 向量存储相关(使用 FAISS)
+ # Vector index using FAISS for semantic search
self._index = None
- # 内部状态:存储提取的知识
+ # Internal state storing the extracted knowledge
self._data: T = self._data_schema()
self.metadata: Dict[str, Any] = {
"created_at": datetime.now(),
@@ -84,7 +87,7 @@ class BaseKnowledge(ABC, Generic[T]):
@staticmethod
def _default_prompt() -> str:
- """默认提取提示词"""
+ """Returns the default extraction prompt template."""
return (
"You are an expert knowledge extraction assistant. "
"Your task is to carefully analyze the following text and extract structured information "
@@ -93,157 +96,173 @@ class BaseKnowledge(ABC, Generic[T]):
"### Source Text:\n"
)
- # ==================== 数据管理 ====================
+ # ==================== Data Management ====================
@property
def data_schema(self) -> Type[T]:
- """
- 返回当前知识类使用的 Schema 类。
+ """Returns the Pydantic schema class used by this knowledge instance.
- :return: Pydantic Schema 类
+ Returns:
+ The Pydantic BaseModel subclass defining the knowledge structure.
"""
return self._data_schema
@property
def data(self) -> T:
- """
- 获取所有存储的知识(只读)。
+ """Returns all stored knowledge (read-only access).
- :return: 内部知识数据
+ Returns:
+ The internal knowledge data as a Pydantic model instance.
"""
return self._data
def clear(self):
- """清空所有知识(数据和索引)。"""
+ """Clears all knowledge including data and vector index."""
self.metadata["updated_at"] = datetime.now()
self._data = self._data_schema()
self.clear_index()
def clear_index(self):
- """清空向量索引(不影响数据)。"""
+ """Clears the vector index without affecting the stored data."""
self._index = None
- # ==================== 提取与聚合 ====================
+ # ==================== Extraction & Aggregation ====================
@abstractmethod
def extract(self, text: str, *, merge_mode: bool = False) -> T:
- """
- 主提取方法 - 自动处理长文本分块、提取、聚合。
-
- 流程:
- 1. 根据 merge_mode 决定是否清空现有数据
- 2. 判断文本长度,决定是否分块
- 3. 对每个块进行提取
- 4. 调用 merge() 合并数据(merge 不修改内部状态)
- 5. extract 负责将 merge 的返回值赋值给 self._data 并调用 clear_index()
- 6. 返回提取到的结果
-
- :param text: 输入文本(可以是短文本或长文本)
- :param merge_mode: 合并模式(默认 False)
- - False(默认): 替换模式 - 仅合并新提取的数据
- - True: 累积模式 - 将现有知识与新数据合并
- :return: 提取到的知识数据
+ """Extracts structured knowledge from text with automatic chunking and aggregation.
+
+ This is the main entry point for knowledge extraction. It handles the complete pipeline
+ from text preprocessing to knowledge aggregation.
+
+ Extraction pipeline:
+ 1. Decide whether to clear existing data based on merge_mode
+ 2. Determine if text chunking is needed based on length
+ 3. Extract knowledge from each chunk using LLM
+ 4. Merge all extracted data using the merge() method
+ 5. Update internal state (_data) and invalidate index via clear_index()
+ 6. Return the aggregated knowledge
+
+ Args:
+ text: Input text to extract knowledge from (can be short or long).
+ merge_mode: Controls how new data is combined with existing knowledge.
+ - False (default): Replace mode - only use newly extracted data
+ - True: Accumulative mode - merge new data with existing knowledge
+
+ Returns:
+ The extracted and aggregated knowledge data.
"""
pass
@abstractmethod
def merge(self, data_list: List[T]) -> T:
- """
- 纯数据合并方法 - 不修改内部状态。
+ """Merges multiple knowledge data objects into a single unified object.
+
+ This is a pure data transformation method that does not modify internal state.
+ Subclasses implement specific merge strategies (deduplication, conflict resolution, etc.).
+
+ Responsibilities:
+ - Implement concrete merge algorithms (deduplication, conflict resolution, etc.)
+ - Return a new merged data object
+ - Never modify instance attributes
- 职责:
- - 实现具体的合并算法(去重、冲突解决等)
- - 返回合并后的新数据对象
- - 不修改任何实例属性
+ Args:
+ data_list: List of knowledge data objects to merge.
- :param data_list: 需要合并的数据列表
- :return: 合并后的知识数据(新对象)
+ Returns:
+ A new merged knowledge object.
"""
pass
- # ==================== 演化 ====================
+ # ==================== Evolution ====================
def evolve(self, **kwargs) -> T:
- """
- 演化内部知识。
- 子类必须实现此方法。
+ """Evolves and optimizes the internal knowledge through inference and refinement.
+
+ Subclasses can implement this method to perform knowledge-specific evolution strategies.
- 职责:
- - 推理隐含关系
- - 剪枝低置信度节点
- - 聚类优化
- - 知识补全
+ Evolution capabilities:
+ - Infer implicit relationships between entities
+ - Prune low-confidence nodes and connections
+ - Apply clustering for better organization
+ - Complete missing knowledge through reasoning
- :return: 新的内部知识数据
+ Returns:
+ The evolved knowledge data.
"""
pass
- # ==================== 索引与查询 ====================
+ # ==================== Indexing & Query ====================
@abstractmethod
def build_index(self):
- """
- 构建或重建向量索引。
+ """Builds or rebuilds the vector index for semantic search.
- 子类必须实现此方法,定义如何构建向量索引。
- 使用 FAISS 作为向量存储后端。
+ Subclasses must implement this method to define how the vector index is constructed
+ from the knowledge data. Uses FAISS as the vector store backend.
"""
pass
@abstractmethod
def search(self, query: str, top_k: int = 3) -> List[Any]:
- """
- 语义搜索知识。
+ """Performs semantic search over the knowledge base.
+
+ Standard search workflow:
+ 1. Ensure index is built (call build_index if needed)
+ 2. Use vector_store.similarity_search for retrieval
+ 3. Restore original data structures from Document.metadata
- 默认实现:
- 1. 确保索引已构建(调用 build_index)
- 2. 使用 vector_store.similarity_search 进行检索
- 3. 从 Document.metadata 恢复原始数据结构
+ Subclasses can override this method to implement custom search logic.
- 子类可覆盖此方法实现自定义搜索逻辑。
+ Args:
+ query: Search query string.
+ top_k: Number of results to return.
- :param query: 查询字符串
- :param top_k: 返回结果数
- :return: 相关知识列表
+ Returns:
+ List of relevant knowledge items.
"""
pass
@abstractmethod
- def size(self) -> int:
- """
- 返回知识点数量。
- 子类必须实现此方法。
+ def __len__(self) -> int:
+ """Returns the number of knowledge items in the collection.
+
+ Subclasses must implement this method to report their specific size metric.
+ This enables using len() function on knowledge instances.
- :return: 知识点数量
+ Returns:
+ The count of knowledge items.
"""
pass
- # ==================== 序列化 ====================
+ # ==================== Serialization ====================
@abstractmethod
def dump(self, folder_path: str | Path):
- """
- 导出知识到指定文件夹。
- 子类必须实现此方法。
+ """Exports knowledge to a specified folder.
+
+ Subclasses must implement this method to persist their knowledge data.
- 需要保存到文件夹内:
- 1. 结构化数据 (self._data) - 保存为 JSON 文件
- 2. 向量索引 (self._index) - FAISS 索引文件
+ Files to save:
+ 1. Structured data (self._data) - saved as JSON file
+ 2. Vector index (self._index) - saved as FAISS index files
- :param folder_path: 目标文件夹路径
+ Args:
+ folder_path: Target folder path for saving knowledge.
"""
pass
@abstractmethod
def load(self, folder_path: str | Path):
- """
- 从文件夹加载知识。
- 子类必须实现此方法。
+ """Loads knowledge from a specified folder.
+
+ Subclasses must implement this method to restore previously saved knowledge.
- 需要从文件夹加载:
- 1. 结构化数据 (self._data) - 从 JSON 文件加载
- 2. 向量索引 (self._index) - 从 FAISS 索引文件加载
+ Files to load:
+ 1. Structured data (self._data) - loaded from JSON file
+ 2. Vector index (self._index) - loaded from FAISS index files
- :param folder_path: 文件夹路径
+ Args:
+ folder_path: Source folder path containing saved knowledge.
"""
pass
diff --git a/src/knowledge/entity.py b/src/knowledge/entity.py
index 10a89dc..e692764 100644
--- a/src/knowledge/entity.py
+++ b/src/knowledge/entity.py
@@ -1,10 +1,8 @@
-"""
-实体知识提取模块。
+"""Entity knowledge extraction module.
-基于 ListKnowledge 的实体提取实现。
+Provides entity extraction implementation based on ListKnowledge.
"""
-from pathlib import Path
from typing import List, Dict, Type, TypeVar, Generic
from pydantic import BaseModel, Field as PydanticField
from langchain_core.language_models.chat_models import BaseChatModel
@@ -23,36 +21,34 @@ except ImportError:
logger = logging.getLogger(__name__)
-# 实体类型变量
+# Entity type variable
E = TypeVar("E", bound="BaseEntitySchema")
class BaseEntitySchema(BaseModel):
- """
- 实体基类 - 所有实体必须包含的核心字段。
+ """Base entity class defining required core fields for all entities.
- 用户可继承此类添加自定义字段。
+ Users can inherit from this class to add custom fields.
"""
- name: str = PydanticField(description="实体唯一名称", min_length=1)
- description: str = PydanticField(description="实体描述")
+ name: str = PydanticField(description="Unique entity name", min_length=1)
+ description: str = PydanticField(description="Entity description")
class Config:
- """允许用户扩展字段"""
+ """Configuration allowing users to extend with additional fields."""
- extra = "allow" # 允许额外字段
- validate_assignment = True # 赋值时验证
+ extra = "allow" # Allow additional fields
+ validate_assignment = True # Validate on assignment
class EntityKnowledge(ListKnowledge[E], Generic[E]):
- """
- 专门用于实体提取和管理的知识类。
+ """Specialized knowledge class for entity extraction and management.
- 特点:
- 1. 自动去重(基于 entity.name)
- 2. LLM 智能合并(同名实体的字段合并)
- 3. 每个实体作为独立文档建索引
- 4. 支持实体级别的搜索
+ Key features:
+ 1. Automatic deduplication based on entity.name
+ 2. Intelligent LLM-powered merging for entities with the same name
+ 3. Each entity indexed as an independent document
+ 4. Support for entity-level semantic search
"""
def __init__(
@@ -64,11 +60,13 @@ class EntityKnowledge(ListKnowledge[E], Generic[E]):
prompt: str = "",
**kwargs,
):
- """
- :param entity_schema: 用户自定义的实体类(必须继承 BaseEntity)
- :param llm_client: LLM 客户端
- :param embedder: 向量化器
- :param prompt: 自定义提示词
+ """Initialize EntityKnowledge with entity schema and configuration.
+
+ Args:
+ entity_schema: User-defined entity class (must inherit from BaseEntity).
+ llm_client: Language model client for extraction and merging.
+ embedder: Embedding model for vector indexing.
+ prompt: Custom extraction prompt.
"""
self._entity_schema = entity_schema
@@ -82,12 +80,12 @@ class EntityKnowledge(ListKnowledge[E], Generic[E]):
self._merge_chain = self._create_merge_chain()
- # 全局实体映射(主存储,类型安全)
+ # Global entity mapping (primary storage, type-safe)
self._entity_map: Dict[str, E] = {}
@staticmethod
def _default_prompt() -> str:
- """默认实体提取提示词"""
+ """Returns the default entity extraction prompt."""
return (
"You are an expert entity extraction assistant. "
"Carefully read the following text and extract all entities "
@@ -97,7 +95,7 @@ class EntityKnowledge(ListKnowledge[E], Generic[E]):
)
def _create_merge_chain(self):
- """创建 LLM 合并 chain,用于智能合并同名实体"""
+ """Creates an LLM chain for intelligently merging entities with the same name."""
merge_template = """You are an expert at merging entity information.
Given two instances of the same entity, intelligently merge their fields into one.
@@ -120,86 +118,89 @@ Please merge these two entities intelligently and return the result."""
self._entity_schema
)
- # ==================== 数据管理 ====================
+ # ==================== Data Management ====================
def clear(self):
- """清空所有实体"""
+ """Clears all entities from storage."""
super().clear()
self._entity_map.clear()
- # ==================== 提取与聚合 ====================
+ # ==================== Extraction & Aggregation ====================
def extract(self, text: str, *, merge_mode: bool = False) -> BaseModel:
- """
- 提取实体知识(支持替换/累积两种模式)。
+ """Extracts entity knowledge with support for replace/accumulate modes.
- 模式说明:
- - merge_mode=False(默认): 替换模式
- 仅使用新提取的实体
+ Mode explanation:
+ - merge_mode=False (default): Replace mode
+ Only uses newly extracted entities
- - merge_mode=True: 累积模式
- 保留现有实体,提取新实体并智能合并(LLM 去重)
+ - merge_mode=True: Accumulate mode
+ Keeps existing entities, extracts new entities and intelligently merges
+ (LLM-powered deduplication)
- :param text: 输入文本
- :param merge_mode: 合并模式(默认 False)
- :return: 提取后的实体列表容器
+ Args:
+ text: Input text to extract entities from.
+ merge_mode: Merge mode (default False).
+
+ Returns:
+ Container with extracted entities.
"""
- # 调用父类提取方法,传递 merge_mode
+ # Call parent extraction method with merge_mode
result = super().extract(text, merge_mode=merge_mode)
- # 提取完成后同步到 entity_map
+ # Synchronize extracted items to entity_map after extraction
self._sync_items_to_map()
logger.info(
- f"Entity extraction completed ({'累积' if merge_mode else '替换'} mode)"
+ f"Entity extraction completed ({'Accumulate' if merge_mode else 'Replace'} mode)"
)
logger.info(f"Total entities: {len(self._entity_map)}")
return result
def merge(self, data_list: List[BaseModel]) -> BaseModel:
- """
- 实体级别的智能合并(使用 LLM)。
+ """Entity-level intelligent merging using LLM.
- 策略:
- 1. 先调用父类 merge 获得所有提取的 items
- 2. 将结果赋值给 self._data
- 3. 同步到 entity_map 进行 LLM 去重
- 4. 返回去重后的数据
+ Strategy:
+ 1. Call parent merge to get all extracted items
+ 2. Assign result to self._data
+ 3. Synchronize to entity_map for LLM-powered deduplication
+ 4. Return deduplicated data
- :param data_list: 多个容器对象
- :return: 合并后的实体数据
+ Args:
+ data_list: Multiple container objects.
+
+ Returns:
+ Merged entity data.
"""
- # 调用父类 merge,获得合并后的结果
+ # Call parent merge to get merged result
merged_result = super().merge(data_list)
- # 临时赋值给 self._data 以便 _sync_items_to_map 能访问 self.items
+ # Temporarily assign to self._data so _sync_items_to_map can access self.items
self._data = merged_result
- # 同步到 entity_map 进行去重
+ # Synchronize to entity_map for deduplication
self._sync_items_to_map()
- # 返回最终的 self._data(已经被 _sync_items_to_map 更新)
+ # Return final self._data (updated by _sync_items_to_map)
return self._data
def _sync_items_to_map(self):
- """
- 将 self.items 同步到 entity_map,执行去重和 LLM 合并。
- """
+ """Synchronizes self.items to entity_map, performing deduplication and LLM merging."""
total_entities = 0
duplicate_count = 0
- # 遍历父类的 items,与已有的 entity_map 合并
+ # Traverse parent's items and merge with existing entity_map
for entity in self.items:
total_entities += 1
entity_name = entity.name.strip()
- if not entity_name: # 跳过空名称
+ if not entity_name: # Skip empty names
logger.warning(f"Skipping entity with empty name: {entity}")
continue
if entity_name in self._entity_map:
- # 实体已存在 - 使用 LLM 执行合并
+ # Entity exists - merge using LLM
duplicate_count += 1
existing = self._entity_map[entity_name]
merged = self._merge_entity_with_llm(existing, entity)
@@ -210,10 +211,10 @@ Please merge these two entities intelligently and return the result."""
f"LLM merge failed for entity '{entity_name}', keeping existing"
)
else:
- # 新实体 - 深拷贝后添加
+ # New entity - deep copy and add
self._entity_map[entity_name] = entity.model_copy(deep=True)
- # 将去重后的结果写回 self.items
+ # Write deduplicated results back to self.items
self._data.items = list(self._entity_map.values())
self.clear_index()
@@ -223,19 +224,21 @@ Please merge these two entities intelligently and return the result."""
)
def _merge_entity_with_llm(self, entity_a: E, entity_b: E) -> E | None:
- """
- 使用 LLM 智能合并两个同名实体。
+ """Uses LLM to intelligently merge two entities with the same name.
- :param entity_a: 第一个实体
- :param entity_b: 第二个实体
- :return: 合并后的实体(失败返回 None)
+ Args:
+ entity_a: First entity.
+ entity_b: Second entity.
+
+ Returns:
+ Merged entity (returns None on failure).
"""
try:
- # 准备输入
+ # Prepare inputs
entity_a_json = entity_a.model_dump_json(indent=2)
entity_b_json = entity_b.model_dump_json(indent=2)
- # 调用 LLM 合并
+ # Call LLM for merging
merged_entity = self._merge_chain.invoke(
{
"entity_a": entity_a_json,
@@ -250,32 +253,31 @@ Please merge these two entities intelligently and return the result."""
logger.error(f"Error merging entity '{entity_a.name}' with LLM: {e}")
return None
- # ==================== 查询与索引 ====================
+ # ==================== Indexing & Query ====================
def build_index(self):
- """
- 为每个实体构建独立的向量索引文档。
+ """Builds independent vector index document for each entity.
- 每个实体 → 一个 Document:
- - page_content: "{name}: {description} + 其他字段"
- - metadata: {"entity": entity.model_dump()}
+ Each entity → one Document:
+ - page_content: "{name}: {description} + other fields"
+ - metadata: {"entity": entity.model_dump()}
"""
if len(self._entity_map) == 0:
logger.warning("No entities to index")
return
if self._index is not None:
- return # 索引已存在
+ return # Index already exists
documents = []
for entity in self._entity_map.values():
- # 构建文档内容(用于向量化)
+ # Build document content for vectorization
content_parts = [
f"name: {entity.name}",
f"description: {entity.description}",
]
- # 添加其他字段(如果用户扩展了)
+ # Add other fields if user extended the schema
for field_name in self._entity_schema.model_fields:
if field_name not in ("name", "description"):
value = getattr(entity, field_name, None)
@@ -284,7 +286,7 @@ Please merge these two entities intelligently and return the result."""
content = "\n".join(content_parts)
- # 保存完整实体数据到 metadata
+ # Save complete entity data to metadata
documents.append(
Document(
page_content=content,
@@ -297,15 +299,17 @@ Please merge these two entities intelligently and return the result."""
def search(self, query: str, top_k: int = 3, return_raw: bool = False) -> List[E]:
- """
- 语义搜索实体。
+ """Performs semantic search over entities.
+
+ Args:
+ query: Search query string.
+ top_k: Number of results to return.
+ return_raw: Whether to return raw entity objects (default returns dict).
- :param query: 查询字符串
- :param top_k: 返回结果数
- :param return_raw: 是否返回原始实体对象(默认返回字典)
- :return: 实体列表
+ Returns:
+ List of entities.
"""
- if self.size() == 0:
+ if len(self) == 0:
logger.warning("No items to search")
return []
@@ -314,17 +318,17 @@ Please merge these two entities intelligently and return the result."""
docs = self._index.similarity_search(query, k=top_k)
- # 恢复实体对象
+ # Restore entity objects
results = []
for doc in docs:
try:
entity_data = doc.metadata["raw"]
if return_raw:
- # 返回 Pydantic 对象
+ # Return Pydantic object
entity = self._entity_schema.model_validate(entity_data)
results.append(entity)
else:
- # 返回字典
+ # Return dictionary
results.append(entity_data)
except Exception as e:
logger.warning(f"Failed to restore entity: {e}")
@@ -332,35 +336,35 @@ Please merge these two entities intelligently and return the result."""
logger.info(f"Found {len(results)} entities for query: {query[:50]}...")
return results
- def size(self) -> int:
- """返回实体数量(直接从 entity_map 读取)"""
+ def __len__(self) -> int:
+ """Returns the number of entities (reads directly from entity_map)."""
return len(self._entity_map)
- # ==================== 便捷属性 ====================
+ # ==================== Convenience Properties ====================
@property
def entities(self) -> List[E]:
- """快速访问实体列表"""
+ """Quick access to entity list."""
return self.items
@property
def entity_schema(self) -> Type[E]:
- """返回用户定义的实体 Schema"""
+ """Returns the user-defined entity schema."""
return self._entity_schema
@property
def entity_names(self) -> List[str]:
- """返回所有实体名称(直接从 entity_map 读取)"""
+ """Returns all entity names (reads directly from entity_map)."""
return list(self._entity_map.keys())
- # ==================== 可选增强方法 ====================
+ # ==================== Optional Enhancement Methods ====================
def get_entity_by_name(self, name: str) -> E | None:
- """按名称获取实体(O(1) 字典查找)"""
+ """Retrieves entity by name (O(1) dictionary lookup)."""
return self._entity_map.get(name)
def remove_entity(self, name: str) -> bool:
- """删除指定实体(从 entity_map 和 items 删除)"""
+ """Removes specified entity (from both entity_map and items)."""
if name in self._entity_map:
del self._entity_map[name]
self._data.items = list(self._entity_map.values())
@@ -369,13 +373,13 @@ Please merge these two entities intelligently and return the result."""
return True
return False
- # ==================== 序列化 ====================
+ # ==================== Serialization ====================
def load(self, folder_path: str, **kwargs):
- """加载后重建 entity_map"""
+ """Loads knowledge and rebuilds entity_map."""
super().load(folder_path, **kwargs)
- # 从 items 重建 entity_map
+ # Rebuild entity_map from items
self._entity_map = {
entity.name: entity.model_copy(deep=True) for entity in self.items
}
diff --git a/src/knowledge/generic.py b/src/knowledge/generic.py
index 1a7d8d4..6f50533 100644
--- a/src/knowledge/generic.py
+++ b/src/knowledge/generic.py
@@ -1,9 +1,10 @@
-"""
-基础知识模式实现。
+"""Fundamental knowledge pattern implementations.
-包含两种核心模式:
-- UnitKnowledge: 单元知识模式(适用于摘要、元数据等)
-- ListKnowledge: 列表知识模式(适用于实体、事件等列表提取)
+This module provides two core knowledge patterns:
+ - UnitKnowledge: Single-object pattern for extracting one structured object per document
+ (suitable for summaries, metadata, etc.)
+ - ListKnowledge: Collection pattern for extracting multiple items from text
+ (suitable for entities, events, references, etc.)
"""
import json
@@ -25,20 +26,24 @@ except ImportError:
logger = logging.getLogger(__name__)
-# ==================== 两种基础知识模式 ====================
+# ==================== Two Fundamental Knowledge Patterns ====================
-# UnitKnowledge: 单元知识模式
+# UnitKnowledge: Single-object knowledge pattern
class UnitKnowledge(BaseKnowledge[T]):
- """
- 单元知识模式 (Unit Knowledge Pattern) - 适用于任何 Pydantic Schema。
-
- 特点:
- - 针对整篇文档提取 **唯一** 的一个结构化对象
- - 合并策略:字段级更新 (Upsert/Patch)
- - 索引策略:对对象的每个非空字段建立索引
- - 使用 LangChain 原生批处理(batch_as_completed)
+ """Unit Knowledge Pattern - extracts a single structured object from text.
+
+ This pattern is designed for extracting exactly one structured object per document,
+ regardless of document length. Suitable for document-level information like summaries,
+ metadata, or aggregate statistics.
+
+ Key characteristics:
+ - Extraction target: One unique structured object per document
+ - Merge strategy: Field-level update (Upsert/Patch) - first extraction wins,
+ subsequent extractions only fill missing fields
+ - Indexing strategy: Each non-null field of the object is indexed independently
+ - Processing: Uses LangChain native batch processing for efficient multi-chunk handling
"""
def __init__(
@@ -54,15 +59,17 @@ class UnitKnowledge(BaseKnowledge[T]):
show_progress: bool = True,
**kwargs,
):
- """
- :param data_schema: 数据 Schema 类(必须是 Pydantic BaseModel 子类)
- :param llm_client: LLM 客户端
- :param embedder: 向量化器
- :param prompt: 自定义提示词
- :param chunk_size: 长文本分块大小
- :param chunk_overlap: 分块重叠大小
- :param max_workers: 最大并发数
- :param show_progress: 是否显示进度
+ """Initialize UnitKnowledge with schema and configuration.
+
+ Args:
+ data_schema: Pydantic BaseModel subclass defining the object structure.
+ llm_client: Language model client for extraction.
+ embedder: Embedding model for vector indexing.
+ prompt: Custom extraction prompt (defaults to generic prompt).
+ chunk_size: Maximum characters per chunk for long texts.
+ chunk_overlap: Overlapping characters between adjacent chunks.
+ max_workers: Maximum concurrent extraction tasks.
+ show_progress: Whether to log progress information.
"""
super().__init__(
data_schema,
@@ -78,7 +85,7 @@ class UnitKnowledge(BaseKnowledge[T]):
@staticmethod
def _default_prompt() -> str:
- """默认提取提示词"""
+ """Returns the default extraction prompt for single-object extraction."""
return (
"You are an expert knowledge extraction assistant. "
"Your task is to carefully analyze the following text and extract structured information "
@@ -87,27 +94,32 @@ class UnitKnowledge(BaseKnowledge[T]):
"### Source Text:\n"
)
- # ==================== 提取与聚合 ====================
+ # ==================== Extraction & Aggregation ====================
def extract(self, text: str, *, merge_mode: bool = False) -> T:
- """
- 使用 LangChain 原生批处理提取知识
+ """Extracts knowledge using LangChain native batch processing.
+
+ Automatically handles text chunking for long documents and aggregates results
+ using the field-level merge strategy.
- :param text: 输入文本
- :param merge_mode: 合并模式(默认 False)
- - False: 替换模式 - 清空后仅使用新提取的数据
- - True: 累积模式 - 将现有数据与新数据合并
- :return: 提取到的知识数据
+ Args:
+ text: Input text to extract knowledge from.
+ merge_mode: Controls data combination behavior.
+ - False (default): Replace mode - clears existing data before extraction
+ - True: Accumulative mode - merges new data with existing knowledge
+
+ Returns:
+ The extracted knowledge object.
"""
start_time = datetime.now()
- # 替换模式:先清空
+ # In replace mode, clear existing data first
if not merge_mode:
self.clear()
- # 判断是否需要分块
+ # Determine if chunking is needed based on text length
if len(text) <= self.chunk_size:
- # 短文本:直接提取
+ # Short text: direct extraction without chunking
if self.show_progress:
logger.info(f"Processing single text (length: {len(text)})...")
@@ -115,7 +127,7 @@ class UnitKnowledge(BaseKnowledge[T]):
extracted_data_list = [extracted_data]
else:
- # 长文本:分块提取
+ # Long text: extract by chunking
chunks = self.text_splitter.split_text(text)
logger.info(f"Split text into {len(chunks)} chunks")
@@ -136,7 +148,7 @@ class UnitKnowledge(BaseKnowledge[T]):
self._data = self.merge(extracted_data_list)
- # 统一修改状态
+ # Update state uniformly
self.clear_index()
self.metadata["updated_at"] = datetime.now()
@@ -147,14 +159,16 @@ class UnitKnowledge(BaseKnowledge[T]):
return self.data
def merge(self, data_list: List[T]) -> T:
- """
- 纯数据合并方法 - 字段级更新策略。
+ """Pure data merge method implementing field-level update strategy.
+
+ Merge strategy: First extraction takes precedence. Subsequent extractions only fill
+ missing fields without overwriting existing values. Implemented using model_copy(update=...).
- 合并策略:第一次提取的结果为准,后续只能补充缺失字段。
- 使用 model_copy(update=...) 实现第一次优先合并。
+ Args:
+ data_list: List of extracted data objects to merge.
- :param data_list: 需要合并的数据列表
- :return: 合并后的知识数据(新对象)
+ Returns:
+ A new merged knowledge object.
"""
result = data_list[0].model_copy()
@@ -163,12 +177,12 @@ class UnitKnowledge(BaseKnowledge[T]):
return result
- # ==================== 查询与索引 ====================
+ # ==================== Indexing & Query ====================
def build_index(self):
- """从 _data 的所有字段构建索引"""
- # 检查是否有数据
- if self.size() == 0:
+ """Builds vector index from all non-null fields in the data object."""
+ # Check if there's data to index
+ if len(self) == 0:
logger.warning("No data to index")
return
@@ -190,15 +204,17 @@ class UnitKnowledge(BaseKnowledge[T]):
logger.info(f"Built FAISS index with {len(documents)} documents")
def search(self, query: str, top_k: int = 3) -> List[Any]:
- """
- 搜索所有列表字段。
+ """Searches all indexed fields using semantic similarity.
+
+ Args:
+ query: Search query string.
+ top_k: Number of results to return.
- :param query: 查询字符串
- :param top_k: 返回结果数
- :return: 相关知识列表(单个 item,不是容器)
+ Returns:
+ List of relevant knowledge items (field-value dictionaries).
"""
- # 检查是否有数据
- if self.size() == 0:
+ # Check if there's data to search
+ if len(self) == 0:
logger.warning("No items to search")
return []
@@ -207,7 +223,7 @@ class UnitKnowledge(BaseKnowledge[T]):
docs = self._index.similarity_search(query, k=top_k)
- # 恢复原始对象
+ # Restore original objects from metadata
results = []
for doc in docs:
try:
@@ -219,21 +235,21 @@ class UnitKnowledge(BaseKnowledge[T]):
logger.info(f"Found {len(results)} results for query: {query[:50]}...")
return results
- def size(self) -> int:
- """返回字段数"""
- return len(self.data_schema.model_fields)
+ def __len__(self) -> int:
+ """Returns 1 since UnitKnowledge represents a single knowledge unit."""
+ return 1
- # ==================== 序列化 ====================
+ # ==================== Serialization ====================
def dump(self, folder_path: str | Path):
- """
- 导出知识到指定文件夹。
+ """Exports knowledge to a specified folder.
- 保存到文件夹内:
- 1. 结构化数据 (self._data) - 保存为 state.json
- 2. 向量索引 (self._index) - FAISS 索引文件
+ Saves to the folder:
+ 1. Structured data (self._data) - saved as state.json
+ 2. Vector index (self._index) - saved as FAISS index files
- :param folder_path: 目标文件夹路径
+ Args:
+ folder_path: Target folder path for saving.
"""
folder = Path(folder_path)
@@ -245,7 +261,7 @@ class UnitKnowledge(BaseKnowledge[T]):
folder.mkdir(parents=True, exist_ok=True)
try:
- # 1. 保存结构化数据
+ # 1. Save structured data
data = {
"schema_name": self.data_schema.__name__,
"data_schema": self.data_schema.model_json_schema(),
@@ -262,7 +278,7 @@ class UnitKnowledge(BaseKnowledge[T]):
f.write(json_str)
logger.info(f"Saved data to {data_file}")
- # 2. 保存向量索引
+ # 2. Save vector index
if self._index is not None:
index_path = str(folder / "faiss_index")
self._index.save_local(index_path)
@@ -274,20 +290,20 @@ class UnitKnowledge(BaseKnowledge[T]):
logger.error(f"Failed to dump knowledge: {e}")
def load(self, folder_path: str | Path):
- """
- 从文件夹加载知识。
+ """Loads knowledge from a specified folder.
- 从文件夹加载:
- 1. 结构化数据 (self._data) - 从 state.json 加载
- 2. 向量索引 (self._index) - 从 FAISS 索引文件加载
+ Loads from the folder:
+ 1. Structured data (self._data) - loaded from state.json
+ 2. Vector index (self._index) - loaded from FAISS index files
- :param folder_path: 文件夹路径
+ Args:
+ folder_path: Source folder path containing saved knowledge.
"""
folder = Path(folder_path)
if not folder.is_dir():
raise ValueError(f"Folder does not exist: {folder_path}")
- # 1. 加载结构化数据
+ # 1. Load structured data
data_file = folder / "state.json"
if not data_file.is_file():
raise ValueError(f"Data file not found: {data_file}")
@@ -298,12 +314,12 @@ class UnitKnowledge(BaseKnowledge[T]):
self._data = self.data_schema.model_validate(data.get("data", {}))
- # 更新元数据
+ # Update metadata with loaded values
if "metadata" in data:
self.metadata.update(data["metadata"])
self.metadata["updated_at"] = datetime.now()
- # 2. 加载向量索引
+ # 2. Load vector index
index_path = str(folder / "faiss_index")
if Path(index_path).exists():
try:
@@ -320,36 +336,38 @@ class UnitKnowledge(BaseKnowledge[T]):
logger.warning("No index file found, will rebuild on next search")
self._index = None
- logger.info(f"Loaded knowledge successfully with {self.size()} fields")
+ logger.info(f"Loaded knowledge successfully with {len(self)} unit(s)")
-# ==================== 列表知识模式 ===================
+# ==================== List Knowledge Pattern ===================
Item = TypeVar("Item", bound=BaseModel)
class ItemListSchema(BaseModel, Generic[Item]):
- """列表容器的通用 Schema"""
+ """Generic schema container for list-based knowledge patterns."""
items: List[Item] = Field(default_factory=list, description="Item list")
class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
- """
- 列表知识模式 (List Collection Pattern)。
+ """List Knowledge Pattern - extracts a collection of objects from text.
+
+ This pattern extracts multiple independent objects from a document, suitable for
+ extracting entities, events, references, or any collection of structured items.
- 特点:
- - 针对整篇文档提取 **一组** 对象列表(如实体、事件、引用等)
- - 合并策略:追加 (Append) + 基础去重(可由子类扩展)
- - 索引策略:对列表中的每个 Item 独立建立索引
+ Key characteristics:
+ - Extraction target: A collection of structured objects
+ - Merge strategy: Append with basic deduplication (extensible by subclasses)
+ - Indexing strategy: Each item in the list is indexed independently
- 与 UnitKnowledge 的区别:
- - UnitKnowledge: 提取单个结构化对象(如摘要、元数据)
- - ListKnowledge: 提取多个独立对象的列表(如实体列表、事件列表)
+ Comparison with UnitKnowledge:
+ - UnitKnowledge: Extracts a single structured object (e.g., summary, metadata)
+ - ListKnowledge: Extracts multiple independent objects (e.g., entity list, event list)
"""
if TYPE_CHECKING:
- # 类型检查时使用泛型版本,保持完整的类型提示
+ # Use generic version during type checking to maintain complete type hints
item_list_schema: Type[ItemListSchema[Item]]
def __init__(
@@ -365,15 +383,17 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
show_progress: bool = True,
**kwargs,
):
- """
- :param item_schema: 列表中单个元素的 Schema 类
- :param llm_client: LLM 客户端
- :param embedder: 向量化器
- :param prompt: 自定义提示词
- :param chunk_size: 长文本分块大小
- :param chunk_overlap: 分块重叠大小
- :param max_workers: 最大并发数
- :param show_progress: 是否显示进度
+ """Initialize ListKnowledge with item schema and configuration.
+
+ Args:
+ item_schema: Pydantic BaseModel subclass for individual list items.
+ llm_client: Language model client for extraction.
+ embedder: Embedding model for vector indexing.
+ prompt: Custom extraction prompt (defaults to list-oriented prompt).
+ chunk_size: Maximum characters per chunk for long texts.
+ chunk_overlap: Overlapping characters between adjacent chunks.
+ max_workers: Maximum concurrent extraction tasks.
+ show_progress: Whether to log progress information.
"""
self.item_schema = item_schema
@@ -400,7 +420,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
@staticmethod
def _default_prompt() -> str:
- """默认列表提取提示词"""
+ """Returns the default extraction prompt for list-based extraction."""
return (
"You are an expert knowledge extraction assistant. "
"Extract all relevant items from the text into a list. "
@@ -411,28 +431,30 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
@property
def items(self) -> List[Item]:
- """获取内部列表"""
+ """Returns the internal list of extracted items."""
return getattr(self._data, "items", [])
def extract(self, text: str, *, merge_mode: bool = False) -> ItemListSchema:
- """
- 使用 LangChain 原生批处理提取列表
+ """Extracts a list of items using LangChain native batch processing.
+
+ Args:
+ text: Input text to extract items from.
+ merge_mode: Controls data combination behavior.
+ - False (default): Replace mode - clears existing data before extraction
+ - True: Accumulative mode - merges new items with existing collection
- :param text: 输入文本
- :param merge_mode: 合并模式(默认 False)
- - False: 替换模式 - 清空后仅使用新提取的数据
- - True: 累积模式 - 将现有数据与新数据合并
- :return: 提取到的 items 列表
+ Returns:
+ The list of extracted items.
"""
start_time = datetime.now()
- # 替换模式:先清空
+ # Replace mode: clear first
if not merge_mode:
self.clear()
- # 判断是否需要分块
+ # Determine if chunking is needed
if len(text) <= self.chunk_size:
- # 短文本:一次性提取
+ # Short text: extract in one pass
if self.show_progress:
logger.info(f"Processing single text (length: {len(text)})...")
@@ -440,7 +462,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
extracted_data_list = [extracted_data]
else:
- # 长文本:分块提取
+ # Long text: extract by chunking
chunks = self.text_splitter.split_text(text)
logger.info(f"Split text into {len(chunks)} chunks")
@@ -461,7 +483,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
self._data = self.merge(extracted_data_list)
- # 统一修改状态
+ # Update state uniformly
self.clear_index()
self.metadata["updated_at"] = datetime.now()
@@ -472,27 +494,30 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
return self.items
def merge(self, data_list: List[ItemListSchema]) -> ItemListSchema:
- """
- 纯数据合并方法 - 列表追加策略。
+ """Pure data merge method implementing list append strategy.
+
+ Merge strategy: Collects all items from all container objects and merges them
+ into a single list. Subclasses can override this method to implement more
+ sophisticated deduplication logic (e.g., EntityKnowledge).
- 合并策略:收集所有容器对象中的 items,合并到一个大列表中。
- 子类可以重写此方法实现更复杂的去重逻辑(如 EntityKnowledge)。
+ Args:
+ data_list: List of container objects to merge.
- :param data_list: 需要合并的容器对象列表
- :return: 合并后的知识数据(新的 ItemListSchema 对象)
+ Returns:
+ A new merged ItemListSchema object.
"""
all_items = []
- # 收集所有 items
+ # Collect all items from each container
for data in data_list:
copied_data = data.model_copy(deep=True)
all_items.extend(copied_data.items)
- # 返回新的对象
+ # Return a new container object
return self.item_list_schema(items=all_items)
def build_index(self):
- """为列表中的每个 Item 构建独立索引"""
+ """Builds independent vector index for each item in the list."""
items = self.items
if not items:
logger.warning("No items to index")
@@ -503,8 +528,8 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
documents = []
for idx, item in enumerate(items):
- # 序列化每个 Item 为 Document
- content = item.model_dump_json() # 使用 JSON 字符串作为内容
+ # Serialize each Item as a Document
+ content = item.model_dump_json() # Use JSON string as content
documents.append(
Document(
page_content=content,
@@ -521,12 +546,14 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
raise
def search(self, query: str, top_k: int = 3) -> List[Any]:
- """
- 搜索列表中的 items。
+ """Searches items in the list using semantic similarity.
- :param query: 查询字符串
- :param top_k: 返回结果数
- :return: 相关的 item 列表
+ Args:
+ query: Search query string.
+ top_k: Number of results to return.
+
+ Returns:
+ List of relevant items.
"""
if not self.items:
logger.warning("No items to search")
@@ -538,7 +565,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
docs = self._index.similarity_search(query, k=top_k)
results = []
for doc in docs:
- # 尝试还原为对象
+ # Attempt to restore as object
try:
raw = doc.metadata.get("raw", {})
item = self.item_schema.model_validate(raw)
@@ -550,21 +577,22 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
logger.info(f"Found {len(results)} results for query: {query[:50]}...")
return results
- def size(self) -> int:
- """返回列表中的元素数量"""
+ def __len__(self) -> int:
+ """Returns the number of elements in the list."""
return len(self.items)
def dump(self, folder_path: str) -> Any:
- """
- 导出知识到指定文件夹(复用 UnitKnowledge 的实现)。
- 容器本质也是 BaseModel,可以直接使用相同的序列化逻辑。
+ """Exports knowledge to a specified folder.
+
+ Uses the same serialization logic as UnitKnowledge since the container
+ is also a BaseModel.
"""
from pathlib import Path
folder = Path(folder_path)
folder.mkdir(parents=True, exist_ok=True)
- # 1. 保存结构化数据
+ # 1. Save structured data
data = {
"schema_name": self.item_list_schema.__name__,
"item_schema_name": self.item_schema.__name__,
@@ -582,7 +610,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
f.write(json_str)
logger.info(f"Saved data to {data_file}")
- # 2. 保存向量索引
+ # 2. Save vector index
if self._index is not None:
index_path = str(folder / "faiss_index")
self._index.save_local(index_path)
@@ -593,14 +621,14 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
return json_str
def load(self, folder_path: str, **kwargs):
- """从文件夹加载知识"""
+ """Loads knowledge from a specified folder."""
from pathlib import Path
folder = Path(folder_path)
if not folder.is_dir():
raise ValueError(f"Folder does not exist: {folder_path}")
- # 1. 加载结构化数据
+ # 1. Load structured data
data_file = folder / "state.json"
if not data_file.is_file():
raise ValueError(f"Data file not found: {data_file}")
@@ -611,12 +639,12 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
self._data = self.item_list_schema.model_validate(data.get("data", {}))
- # 更新元数据
+ # Update metadata with loaded values
if "metadata" in data:
self.metadata.update(data["metadata"])
self.metadata["updated_at"] = datetime.now()
- # 2. 加载向量索引
+ # 2. Load vector index
index_path = str(folder / "faiss_index")
if Path(index_path).exists():
try:
@@ -631,7 +659,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):
logger.warning("No index file found, will rebuild on next search")
self._index = None
- logger.info(f"Loaded knowledge successfully with {self.size()} items")
+ logger.info(f"Loaded knowledge successfully with {len(self)} items")
# Set Knowledge