Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
添加知识向量搜索功能示例,重构知识类以支持向量化提取和存储
yifanfeng97f851e1447a59
diff --git a/examples/knowledge_vector_search_demo.py b/examples/knowledge_vector_search_demo.pynew file mode 100644index 0000000..dddca00--- /dev/null+++ b/examples/knowledge_vector_search_demo.py@@ -0,0 +1,125 @@+"""+GenericKnowledge 向量搜索功能演示++展示如何使用新的向量索引功能进行语义搜索。+"""++from pydantic import BaseModel, Field+from typing import List+++# ==================== 1. 定义 Schema ====================++class Article(BaseModel):+ """文章 Schema"""+ title: str = Field(description="文章标题")+ author: str = Field(description="作者")+ content: str = Field(description="文章内容摘要")+ tags: List[str] = Field(default_factory=list, description="标签")+++# ==================== 2. 初始化 Knowledge ====================++async def main():+ from langchain_openai import ChatOpenAI, OpenAIEmbeddings+ from src.knowledge.generic import GenericKnowledge++ # 初始化+ knowledge = GenericKnowledge(+ llm_client=ChatOpenAI(model="gpt-4"),+ embedder=OpenAIEmbeddings(),+ schema_class=Article,+ prompt="Extract article information:",+ max_workers=5,+ show_progress=True+ )++ # ==================== 3. 提取知识 ====================++ text = """+ 人工智能的发展历程+ 作者:张三++ 人工智能(AI)从1956年达特茅斯会议诞生至今,经历了多次起伏。+ 深度学习的突破使得AI在图像识别、自然语言处理等领域取得巨大进展。++ 大语言模型的崛起+ 作者:李四++ 2023年,ChatGPT的发布标志着大语言模型时代的到来。+ 这些模型能够理解和生成人类语言,应用范围广泛。+ """++ stats = knowledge.extract(text)+ print(f"✅ 提取完成:{stats}")+ print(f"📊 共提取 {knowledge.size()} 篇文章\n")++ # ==================== 4. 向量搜索 ====================++ print("🔍 开始语义搜索...")++ # 搜索 1:关于深度学习+ results = knowledge.search("深度学习和神经网络", top_k=2)+ print("\n查询:'深度学习和神经网络'")+ for i, article in enumerate(results, 1):+ print(f" {i}. {article.title} - {article.author}")++ # 搜索 2:关于 ChatGPT+ results = knowledge.search("ChatGPT 语言模型", top_k=2)+ print("\n查询:'ChatGPT 语言模型'")+ for i, article in enumerate(results, 1):+ print(f" {i}. {article.title} - {article.author}")++ # ==================== 5. 持久化 ====================++ # 保存+ knowledge.dump("knowledge_data.json")+ print("\n💾 已保存到 knowledge_data.json")++ # 清空并重新加载+ knowledge.clear()+ print(f"🗑️ 清空后数量:{knowledge.size()}")++ knowledge.load("knowledge_data.json")+ print(f"📥 加载后数量:{knowledge.size()}")++ # 加载后搜索仍然有效(索引会自动重建)+ results = knowledge.search("AI 发展", top_k=1)+ print(f"\n🔍 加载后搜索:找到 {len(results)} 条结果")+++# ==================== 6. 架构说明 ====================++"""+## 📐 架构设计++### BaseKnowledge(模板方法模式)+- `_vector_store`: 向量数据库实例(FAISS)+- `_index_dirty`: 脏标记,追踪数据变化+- `build_index()`: 模板方法,统一索引构建流程+- `_data_to_documents()`: 抽象钩子,子类实现转换逻辑++### GenericKnowledge(具体实现)+1. **_data_to_documents()**: 将 self._items 转为 Document 列表+2. **merge()**: 添加数据后设置 `self._index_dirty = True`+3. **search()**:+ - 调用 `build_index()` 确保索引最新+ - 使用 `_vector_store.similarity_search()`+ - 从 metadata 恢复原始对象+4. **clear()**: 清空数据并重置索引++### 工作流程+```+提取 → merge (设置 dirty=True) → search (自动 build_index) → 返回结果+```++### 优势+✅ 延迟索引:只在需要时构建+✅ 自动重建:数据变化后搜索时自动更新+✅ 可插拔后端:FAISS/Qdrant/Chroma 等+✅ 持久化友好:load 后自动标记 dirty+"""++if __name__ == "__main__":+ import asyncio+ asyncio.run(main())diff --git a/src/knowledge/base.py b/src/knowledge/base.pyindex f2a4579..81cbb28 100644--- a/src/knowledge/base.py+++ b/src/knowledge/base.py@@ -1,329 +1,226 @@-import asynciofrom abc import ABC, abstractmethod-from typing import TypeVar, Generic, Any, Dict, Optional, Type, List, Callable+from pathlib import Path+from typing import TypeVar, Generic, Any, Dict, Type, Listfrom datetime import datetimefrom pydantic import BaseModelfrom langchain_core.language_models.chat_models import BaseChatModelfrom langchain_core.embeddings import Embeddings-try:- from src.config import logger-except ImportError:- import logging- logger = logging.getLogger(__name__)-T = TypeVar("T", bound=BaseModel)-# ===================== 提取器 (Extractor) =====================+# ===================== 知识类 (Knowledge) =====================-class BaseExtractor(ABC, Generic[T]):+class BaseKnowledge(ABC, Generic[T]):"""- 矿工 - 无状态提取器基类。-- 职责:从文本中提取结构化知识,不负责存储或融合。- """-- def __init__(self, llm_client: BaseChatModel, schema_class: Type[T]):- """- :param llm_client: LangChain 的 ChatModel 实例- :param schema_class: 期望的返回 Schema 类(必须继承 BaseModel)- """- self.llm_client = llm_client- self.schema_class = schema_class- @abstractmethod- async def aparse(self, text: str, **kwargs) -> T:- """- 异步解析文本,返回单个知识对象。-- :param text: 输入文本- :return: 解析后的知识对象- """- pass-- def parse(self, text: str, **kwargs) -> T:- """- 同步解析(子类可覆盖为真正的同步实现)。-- 默认行为:需要在异步上下文中调用 aparse。- """- try:- loop = asyncio.get_event_loop()- if loop.is_running():- raise RuntimeError("Cannot use parse() in async context. Use aparse() instead.")- return loop.run_until_complete(self.aparse(text, **kwargs))- except RuntimeError:- raise-- async def batch_parse(self, texts: List[str], **kwargs) -> List[T]:- """- 批量解析多个文本。-- :param texts: 文本列表- :return: 解析结果列表- """- tasks = [self.aparse(text, **kwargs) for text in texts]- return await asyncio.gather(*tasks)-- async def batch_parse_concurrent(- self,- texts: List[str],- max_concurrent: int = 5,- **kwargs- ) -> List[T]:- """- 带并发限制的批量解析。-- :param texts: 文本列表- :param max_concurrent: 最大并发数- :return: 解析结果列表- """- from asyncio import Semaphore-- semaphore = Semaphore(max_concurrent)-- async def limited_parse(text: str) -> T:- async with semaphore:- return await self.aparse(text, **kwargs)-- tasks = [limited_parse(text) for text in texts]- return await asyncio.gather(*tasks)------# ===================== 集合/容器 (Collection) =====================---class BaseCollection(ABC, Generic[T]):- """- 仓库 - 有状态集合基类。-- 职责:存储知识点、融合去重、演化优化、序列化。+ 统一的知识基类 - 集提取、存储、聚合、演化于一体。++ 职责:+ 1. 从文本中提取结构化知识(extract)+ 2. 自动处理长文本分块+ 3. 存储和聚合提取的知识+ 4. 演化优化内部知识+ 5. 序列化与反序列化"""def __init__(self,llm_client: BaseChatModel,embedder: Embeddings,- schema_class: Type[T],- storage: str = "memory",+ *,+ prompt: str = "",+ chunk_size: int = 2000,+ chunk_overlap: int = 200,+ max_workers: int = 10,+ show_progress: bool = True,**kwargs,):"""- :param llm_client: LLM 实例(用于演化等操作)- :param embedder: 向量化器实例(用于语义搜索等)- :param schema_class: 容器持有的知识 Schema 类- :param storage: 存储后端类型("memory"、"postgres" 等)+ 初始化知识对象。++ :param llm_client: LLM 客户端(用于提取和演化)+ :param embedder: 向量化器(用于语义搜索和相似度计算)+ :param prompt: 用户自定义的提示词(用于 extract)+ :param chunk_size: 长文本分块大小+ :param chunk_overlap: 分块重叠大小+ :param max_workers: 最大并发数(用于并发提取)+ :param show_progress: 是否显示进度信息"""self.llm_client = llm_clientself.embedder = embedder- self.schema_class = schema_class- self.storage = storage- self.metadata: Dict[str, Any] = {}-- # ================= 存储层 =================+ self.chunk_size = chunk_size+ self.chunk_overlap = chunk_overlap+ self.prompt = prompt+ self.max_workers = max_workers+ self.show_progress = show_progress++ # 向量存储相关(使用 FAISS)+ self._index = None+ self._index_dirty: bool = True # 标记索引是否需要重建++ # 内部状态:存储提取的知识+ self._data: T = self._init_data()+ self.metadata: Dict[str, Any] = {+ "created_at": datetime.now(),+ "updated_at": datetime.now(),+ "total_extractions": 0,+ "total_chunks_processed": 0,+ }- @abstractmethod- async def add(self, item_id: str, item: T, **kwargs):+ @property+ def schema_class(self) -> Type[T]:"""- 添加知识点。-- :param item_id: 唯一标识符- :param item: 知识对象+ 返回当前知识类使用的 Schema 类。+ 子类可覆盖此属性,或在泛型参数中指定。"""- pass+ # 默认实现:从泛型参数推导(仅当子类未覆盖时)+ raise NotImplementedError("schema_class must be implemented by subclass")@abstractmethod- async def remove(self, item_id: str, **kwargs) -> bool:+ def _init_data(self) -> T:"""- 删除知识点。-- :return: 是否成功删除+ 初始化内部数据结构。++ :return: 初始化的知识容器"""- pass+ return self.schema_class()++ # ==================== 向量索引管理(模板方法模式)====================@abstractmethod- async def update(self, item_id: str, item: T, **kwargs):+ def build_index(self):"""- 更新知识点。-- :param item_id: 知识点 ID- :param item: 新的知识对象+ 构建或重建向量索引。++ 子类必须实现此方法,定义如何构建向量索引。+ 使用 FAISS 作为向量存储后端。"""pass@abstractmethod- def get(self, item_id: str) -> Optional[T]:- """获取单个知识点"""+ def clear(self):+ """清空所有知识。"""pass- @abstractmethod- def list_all(self) -> List[T]:- """列出所有知识点"""- pass+ # ==================== 核心方法:extract ====================@abstractmethod- def search(self, filter_fn: Callable[[T], bool]) -> List[T]:- """- 按条件查询知识点。-- :param filter_fn: 过滤函数+ def extract(self, text: str, **kwargs) -> Dict[str, Any]:"""- pass+ 主提取方法 - 自动处理长文本分块、提取、聚合。- @abstractmethod- def size(self) -> int:- """返回容器中知识点的数量"""+ 流程:+ 1. 判断文本长度,决定是否分块+ 2. 对每个块调用 _extract_chunk() 进行提取+ 3. 调用 merge() 将提取结果聚合到 self._data+ 4. 返回提取统计信息++ :param text: 输入文本(可以是短文本或长文本)+ :return: 提取统计信息 {"chunks_processed": 3, "items_extracted": 15, ...}+ """pass- # ================= 融合层 =================+ # ==================== 存储与聚合 ====================@abstractmethod- async def aggregate(self, items: List[T], **kwargs) -> Dict[str, Any]:+ def merge(self, items: List[T], **kwargs) -> Dict[str, Any]:"""- 融合多个知识点。-- 职责:清洗、去重、合并。-- :param items: 新提取的知识点列表- :return: 融合统计信息(去重数、合并数等)+ 将多个提取结果合并到内部状态 self._data。++ 职责:去重、合并、冲突解决。+ 子类必须实现此方法。++ :param items: 从各个块提取的知识列表+ :return: 合并统计信息 {"items_added": 10, "duplicates_removed": 2, ...}"""pass- async def fuse_batch(self, new_items: List[T], iteration: int = 0) -> Dict[str, Any]:- """- 融合一批新知识点到容器中。-- :param new_items: 新提取的知识点- :param iteration: 迭代号- :return: 融合统计信息+ # ==================== 查询接口 ====================++ @property+ def data(self) -> T:"""- start_time = datetime.now()+ 获取所有存储的知识(只读)。- try:- # 应用融合策略- stats = await self.aggregate(new_items, iteration=iteration)+ :return: 内部知识数据+ """+ return self._data- stats["timestamp"] = start_time.isoformat()- stats["duration_ms"] = (datetime.now() - start_time).total_seconds() * 1000- stats["iteration"] = iteration+ @abstractmethod+ def search(self, query: str, top_k: int = 10, **kwargs) -> List[Any]:+ """+ 语义搜索知识。- logger.info(f"Fused batch: {stats}")- return stats+ 默认实现:+ 1. 确保索引已构建(调用 build_index)+ 2. 使用 vector_store.similarity_search 进行检索+ 3. 从 Document.metadata 恢复原始数据结构- except Exception as e:- logger.error(f"Error during fuse_batch: {e}")- raise+ 子类可覆盖此方法实现自定义搜索逻辑。- # ================= 演化层 =================+ :param query: 查询字符串+ :param top_k: 返回结果数+ :return: 相关知识列表+ """+ pass@abstractmethod- async def evolve(self, **kwargs) -> Dict[str, Any]:+ def size(self) -> int:"""- 演化内部知识。-- 职责:推理新关系、剪枝低置信度、聚类等高级操作。-- :return: 演化统计信息+ 返回知识点数量。+ 子类必须实现此方法。++ :return: 知识点数量"""pass- # ================= 高级流程 =================+ # ==================== 演化 ====================- async def iterative_fuse_and_parse(- self,- extractor: "BaseExtractor[T]",- texts: List[str],- max_iterations: int = 3,- convergence_threshold: float = 0.95,- ) -> Dict[str, Any]:- """- 迭代提取与融合流程。-- 流程:- 1. 第一次提取:从文本中提取所有知识点- 2. 融合去重- 3. 演化优化- 4. 重复直到收敛或达到最大迭代数-- :param extractor: 知识提取器- :param texts: 输入文本列表- :param max_iterations: 最大迭代数- :param convergence_threshold: 收敛阈值- :return: 迭代结果摘要+ @abstractmethod+ def evolve(self, **kwargs) -> T:"""- iteration = 0- previous_size = 0- fusion_stats = []-- while iteration < max_iterations:- logger.info(f"Iteration {iteration}: extracting...")-- # 1. 提取新知识- extracted = await extractor.batch_parse(texts)-- # 2. 融合- logger.info(f"Iteration {iteration}: fusing...")- fuse_result = await self.fuse_batch(extracted, iteration)- fusion_stats.append(fuse_result)-- # 3. 演化- logger.info(f"Iteration {iteration}: evolving...")- evolve_result = await self.evolve(iteration=iteration)-- # 4. 检查收敛- current_size = self.size()- logger.info(f"Iteration {iteration}: size={current_size}")-- if previous_size > 0:- growth_rate = (current_size - previous_size) / previous_size- if growth_rate < (1 - convergence_threshold):- logger.info(- f"Converged at iteration {iteration} (growth_rate={growth_rate})"- )- break-- previous_size = current_size- iteration += 1-- await asyncio.sleep(0.1)-- return {- "final_iteration": iteration,- "total_items": self.size(),- "fusion_history": fusion_stats,- }+ 演化内部知识。+ 子类必须实现此方法。- # ================= 序列化 =================+ 职责:+ - 推理隐含关系+ - 剪枝低置信度节点+ - 聚类优化+ - 知识补全- @abstractmethod- def dump(self, format: str = "json") -> Any:- """- 导出容器内的知识。-- :param format: 格式 ("json", "dict" 等)+ :return: 新的内部知识数据"""pass+ # ==================== 序列化 ====================+@abstractmethod- def load(self, data: Any, **kwargs):- """- 从外部数据恢复知识。-- :param data: 外部数据 (dict, json string 或文件路径)+ def dump(self, folder_path: str | Path) -> Any:"""- pass+ 导出知识到指定文件夹。+ 子类必须实现此方法。- def clear(self):- """清空所有知识点(可选实现)"""- pass+ 需要保存到文件夹内:+ 1. 结构化数据 (self._data) - 保存为 JSON 文件+ 2. 向量索引 (self._index) - FAISS 索引文件+ :param folder_path: 目标文件夹路径+ :return: 序列化后的数据+ """+ pass-# ===================== 通用实现 =====================-+ @abstractmethod+ def load(self, folder_path: str | Path, **kwargs):+ """+ 从文件夹加载知识。+ 子类必须实现此方法。+ 需要从文件夹加载:+ 1. 结构化数据 (self._data) - 从 JSON 文件加载+ 2. 向量索引 (self._index) - 从 FAISS 索引文件加载+ :param folder_path: 文件夹路径+ """+ passdiff --git a/src/knowledge/generic.py b/src/knowledge/generic.pyindex 75af59b..9a2029f 100644--- a/src/knowledge/generic.py+++ b/src/knowledge/generic.py@@ -1,18 +1,20 @@"""通用 Generic 实现。-包含基于 BaseExtractor 和 BaseCollection 的具体实现。+包含基于 BaseKnowledge 的具体实现。"""-import osimport json-from typing import List, Dict, Any, Callable, Optional, Type+import hashlib+from typing import List, Dict, Any, Optional, Typefrom datetime import datetime-from pydantic import BaseModel, ValidationError+from pydantic import ValidationErrorfrom langchain_core.language_models.chat_models import BaseChatModelfrom langchain_core.embeddings import Embeddings+from langchain_core.documents import Document+from langchain_core.vectorstores import VectorStore-from .base import BaseExtractor, BaseCollection, T+from .base import BaseKnowledge, Ttry:from src.config import logger@@ -21,36 +23,14 @@ except ImportError:logger = logging.getLogger(__name__)-class GenericExtractor(BaseExtractor[T]):+class GenericKnowledge(BaseKnowledge[T]):"""- 通用提取器。使用 LLM 的 with_structured_output 实现提取。- """-- async def aparse(self, text: str, **kwargs) -> T:- """- 使用 with_structured_output 异步解析。- """- try:- # 创建支持结构化输出的 LLM- structured_llm = self.llm_client.with_structured_output(self.schema_class)-- # 调用 LLM- result: T = await structured_llm.ainvoke(text)-- logger.info(f"Successfully parsed: {type(result).__name__}")- return result-- except ValidationError as e:- logger.error(f"Schema validation failed: {e}")- raise- except Exception as e:- logger.error(f"Error during parsing: {e}")- raise---class GenericCollection(BaseCollection[T]):- """- 通用集合实现。支持任何 Pydantic BaseModel Schema。+ 通用知识实现 - 适用于任何 Pydantic Schema。++ 特点:+ - 使用 LLM 的 with_structured_output 进行提取+ - 基于 JSON 哈希进行去重+ - 支持向量化语义搜索"""def __init__(@@ -58,147 +38,370 @@ class GenericCollection(BaseCollection[T]):llm_client: BaseChatModel,embedder: Embeddings,schema_class: Type[T],- storage: str = "memory",+ deduplication_strategy: str = "hash",**kwargs,):""":param llm_client: LLM 实例:param embedder: 向量化器实例:param schema_class: Schema 类- :param storage: 存储类型- """- super().__init__(llm_client, embedder, schema_class, storage, **kwargs)- self._items: Dict[str, T] = {}- self._updated_at: Dict[str, datetime] = {}-- # ================= 存储层实现 =================-- async def add(self, item_id: str, item: T, **kwargs):- """添加知识点"""- self._items[item_id] = item- self._updated_at[item_id] = datetime.now()- logger.info(f"Added item: {item_id}")-- async def remove(self, item_id: str, **kwargs) -> bool:- """删除知识点"""- if item_id in self._items:- del self._items[item_id]- del self._updated_at[item_id]- logger.info(f"Removed item: {item_id}")- return True- return False-- async def update(self, item_id: str, item: T, **kwargs):- """更新知识点"""- if item_id in self._items:- self._items[item_id] = item- self._updated_at[item_id] = datetime.now()- logger.info(f"Updated item: {item_id}")- else:- await self.add(item_id, item, **kwargs)-- def get(self, item_id: str) -> Optional[T]:- """获取单个知识点"""- return self._items.get(item_id)+ :param deduplication_strategy: 去重策略("hash" 或 "semantic")+ :param kwargs: 其他参数(prompt, max_workers, show_progress 等)+ """+ super().__init__(llm_client, embedder, **kwargs)+ self._schema_class = schema_class+ self.deduplication_strategy = deduplication_strategy+ self._items: List[T] = [] # 内部存储列表+ self._item_hashes: set = set() # 用于基于哈希的去重++ @property+ def schema_class(self) -> Type[T]:+ """返回当前使用的 Schema 类"""+ return self._schema_class++ def _init_data(self) -> T:+ """初始化为 Schema 类的空实例"""+ try:+ return self.schema_class()+ except Exception:+ return self.schema_class.model_construct()- def list_all(self) -> List[T]:- """列出所有知识点"""- return list(self._items.values())+ def build_index(self):+ """+ 构建或重建向量索引(使用 FAISS)。+ """+ if not self._index_dirty and self._index is not None:+ return # 索引已是最新++ # 1. 将 items 转为 Document 列表+ documents = []+ for i, item in enumerate(self._items):+ documents.append(Document(+ page_content=item.model_dump_json(),+ metadata={+ "index": i,+ "hash": self._compute_hash(item),+ "raw": item.model_dump(),+ }+ ))++ if not documents:+ logger.warning("No documents to index")+ return++ # 2. 使用 FAISS 构建向量存储+ try:+ from langchain_community.vectorstores import FAISS+ self._index = FAISS.from_documents(+ documents=documents,+ embedding=self.embedder+ )+ self._index_dirty = False+ logger.info(f"Built FAISS index with {len(documents)} documents")+ except ImportError:+ logger.error("FAISS not available. Install with: pip install faiss-cpu")+ raise- def search(self, filter_fn: Callable[[T], bool]) -> List[T]:- """按条件查询"""- return [item for item in self._items.values() if filter_fn(item)]+ def extract(self, text: str, **kwargs) -> Dict[str, Any]:+ """主提取方法 - 自动处理长文本分块、提取、聚合"""+ import asyncio+ from datetime import datetime+ start_time = datetime.now()++ # 1. 分块+ chunks = self._chunk_text(text) if len(text) > self.chunk_size else [text]+ if self.show_progress:+ logger.info(f"Processing {len(chunks)} chunk(s) with max_workers={self.max_workers}...")++ # 2. 并发提取+ async def extract_all():+ from asyncio import Semaphore+ semaphore = Semaphore(self.max_workers)++ async def limited_extract(i: int, chunk: str) -> Optional[T]:+ async with semaphore:+ try:+ if self.show_progress:+ logger.info(f"Extracting chunk {i + 1}/{len(chunks)}...")+ return await self._extract_chunk(chunk, **kwargs)+ except Exception as e:+ logger.error(f"Error extracting chunk {i + 1}: {e}")+ return None++ tasks = [limited_extract(i, chunk) for i, chunk in enumerate(chunks)]+ return await asyncio.gather(*tasks)++ # 运行异步提取+ try:+ loop = asyncio.get_event_loop()+ if loop.is_running():+ # 如果已在异步环境中,尝试使用 nest_asyncio(如果可用)+ try:+ import nest_asyncio+ nest_asyncio.apply()+ results = loop.run_until_complete(extract_all())+ except ImportError:+ logger.warning("nest_asyncio not available, creating new event loop")+ loop = asyncio.new_event_loop()+ asyncio.set_event_loop(loop)+ results = loop.run_until_complete(extract_all())+ else:+ results = loop.run_until_complete(extract_all())+ except RuntimeError:+ loop = asyncio.new_event_loop()+ asyncio.set_event_loop(loop)+ results = loop.run_until_complete(extract_all())++ chunk_results = [r for r in results if r is not None]- def size(self) -> int:- """返回知识点数量"""- return len(self._items)+ # 3. 合并+ logger.info("Merging extracted knowledge...")+ aggregate_stats = self.merge(chunk_results, **kwargs)- def clear(self):- """清空所有知识点"""- self._items.clear()- self._updated_at.clear()- logger.info("Cleared all items")+ # 4. 更新元数据+ self.metadata["updated_at"] = datetime.now()+ self.metadata["total_extractions"] += 1+ self.metadata["total_chunks_processed"] += len(chunks)- # ================= 融合层实现 =================+ # 5. 返回统计+ duration = (datetime.now() - start_time).total_seconds()+ return {+ "success": True,+ "chunks_processed": len(chunks),+ "chunks_succeeded": len(chunk_results),+ "aggregate_stats": aggregate_stats,+ "duration_seconds": duration,+ "timestamp": start_time.isoformat(),+ }- async def aggregate(self, items: List[T], **kwargs) -> Dict[str, Any]:+ def _chunk_text(self, text: str) -> List[str]:+ """将长文本分块"""+ try:+ from langchain_text_splitters import RecursiveCharacterTextSplitter+ splitter = RecursiveCharacterTextSplitter(+ chunk_size=self.chunk_size,+ chunk_overlap=self.chunk_overlap,+ separators=["\n\n", "\n", "。", "!", "?", ". ", "! ", "? ", " ", ""],+ )+ chunks = splitter.split_text(text)+ logger.info(f"Split text into {len(chunks)} chunks")+ return chunks+ except ImportError:+ logger.warning("langchain_text_splitters not available, using simple chunking")+ chunks = []+ for i in range(0, len(text), self.chunk_size - self.chunk_overlap):+ chunks.append(text[i:i + self.chunk_size])+ return chunks++ async def _extract_chunk(self, chunk: str, **kwargs) -> T:"""- 默认融合实现:简单去重合并。+ 使用 LLM 的 with_structured_output 提取单个块。- 子类可覆盖此方法实现更复杂的融合逻辑。+ :param chunk: 文本块+ :return: 提取的知识对象"""- logger.info(f"Aggregating {len(items)} items...")-- # 简单的聚合:直接添加到容器- for idx, item in enumerate(items):- item_id = f"item_{len(self._items)}_{idx}"- await self.add(item_id, item)+ try:+ structured_llm = self.llm_client.with_structured_output(self.schema_class)++ # 如果有自定义 prompt,则在消息前加上+ if self.prompt:+ full_text = f"{self.prompt}\n\n{chunk}"+ else:+ full_text = chunk++ result: T = await structured_llm.ainvoke(full_text)+ logger.info(f"Extracted from chunk: {type(result).__name__}")+ return result+ except ValidationError as e:+ logger.error(f"Schema validation failed: {e}")+ raise+ except Exception as e:+ logger.error(f"Error during extraction: {e}")+ raise+ def merge(self, items: List[T], **kwargs) -> Dict[str, Any]:+ """+ 合并提取结果到内部列表。++ :param items: 提取的知识列表+ :return: 合并统计+ """+ added = 0+ duplicates = 0++ for item in items:+ # 计算哈希用于去重+ item_hash = self._compute_hash(item)++ if item_hash not in self._item_hashes:+ self._items.append(item)+ self._item_hashes.add(item_hash)+ added += 1+ logger.debug(f"Added item: {item_hash[:8]}...")+ else:+ duplicates += 1+ logger.debug(f"Skipped duplicate: {item_hash[:8]}...")++ # 更新 self.data(如果 Schema 有 items 字段)+ if hasattr(self._data, "items") and hasattr(self._data.items, "append"):+ # 如果 data.items 是列表,更新它+ self._data.items = self._items++ # 标记索引需要重建+ self._index_dirty = True+return {- "input_count": len(items),- "output_count": len(items),- "deduplicated": 0,+ "items_added": added,+ "duplicates_removed": duplicates,+ "total_items": len(self._items),}- # ================= 演化层实现 =================+ def _compute_hash(self, item: T) -> str:+ """计算知识对象的哈希值"""+ item_json = item.model_dump_json(sort_keys=True)+ return hashlib.sha256(item_json.encode()).hexdigest()- async def evolve(self, **kwargs) -> Dict[str, Any]:+ def search(self, query: str, top_k: int = 10, **kwargs) -> List[T]:+ """+ 基于向量索引的语义搜索。++ :param query: 查询字符串+ :param top_k: 返回结果数+ :return: 最相关的知识列表+ """+ if not self._items:+ logger.warning("No items to search")+ return []++ try:+ # 1. 确保索引已构建+ self.build_index()++ if self._index is None:+ logger.warning("Vector store not available, falling back to text matching")+ raise Exception("Vector store not initialized")++ # 2. 使用向量存储进行语义搜索+ docs = self._index.similarity_search(query, k=top_k)++ # 3. 从 metadata 恢复原始对象+ results = []+ for doc in docs:+ try:+ item = self.schema_class.model_validate(doc.metadata["raw"])+ results.append(item)+ except Exception as e:+ logger.warning(f"Failed to restore item from metadata: {e}")+ continue++ logger.info(f"Found {len(results)} results for query: {query[:50]}...")+ return results++ except Exception as e:+ logger.error(f"Error during vector search: {e}, falling back to text matching")+ # 降级到简单的文本匹配+ return [+ item for item in self._items+ if query.lower() in item.model_dump_json().lower()+ ][:top_k]++ def size(self) -> int:+ """返回知识点数量"""+ return len(self._items)++ async def evolve(self, **kwargs) -> T:"""默认演化实现:空操作。子类可覆盖此方法实现知识演化、推理、剪枝等逻辑。++ :return: 内部知识数据"""logger.debug("Default evolve: no operation")- return {"evolved": False, "message": "Default implementation"}+ return self._data- # ================= 序列化实现 =================+ def get_all_items(self) -> List[T]:+ """获取所有知识项列表"""+ return self._items.copy()- def dump(self, format: str = "json") -> Any:- """导出容器内的知识"""- if not self._items:- logger.warning("No items to dump")- return None+ def dump(self, filepath: str) -> Any:+ """+ 导出知识为 JSON 文件。- try:- if format.lower() == "json":- data = {- k: v.model_dump() for k, v in self._items.items()- }- return json.dumps(data, indent=2, ensure_ascii=False)+ :param filepath: 文件路径+ :return: 序列化后的数据(JSON 字符串)+ """+ from pathlib import Path++ data = {+ "schema_class": self._schema_class.__name__,+ "items": [item.model_dump() for item in self._items],+ "metadata": {k: str(v) if isinstance(v, datetime) else v for k, v in self.metadata.items()},+ }+ json_str = json.dumps(data, indent=2, ensure_ascii=False, default=str)- elif format.lower() == "dict":- return {k: v.model_dump() for k, v in self._items.items()}+ filepath = Path(filepath)+ filepath.parent.mkdir(parents=True, exist_ok=True)++ with open(filepath, "w", encoding="utf-8") as f:+ f.write(json_str)++ logger.info(f"Dumped {len(self._items)} items to {filepath}")+ return json_str- else:- raise ValueError(f"Unsupported format: {format}")+ def load(self, path_or_json: str | dict, **kwargs):+ """+ 加载知识。- except Exception as e:- logger.error(f"Error dumping: {e}")- raise+ :param path_or_json: JSON 文件路径或 JSON 字符串或字典+ """+ from pathlib import Path++ # 解析输入+ if isinstance(path_or_json, dict):+ data = path_or_json+ logger.info("Loading from dictionary")+ elif isinstance(path_or_json, str):+ path = Path(path_or_json)+ if path.is_file():+ with open(path, "r", encoding="utf-8") as f:+ data = json.load(f)+ logger.info(f"Loaded from file: {path}")+ else:+ # 尝试作为 JSON 字符串解析+ data = json.loads(path_or_json)+ logger.info("Loaded from JSON string")+ else:+ raise ValueError(f"Unsupported data type: {type(path_or_json)}")- def load(self, data: Any, **kwargs):- """从外部数据恢复知识"""- try:- if isinstance(data, str):- # 检查是否是文件路径- if os.path.isfile(data):- with open(data, "r", encoding="utf-8") as f:- data = json.load(f)- logger.info(f"Loaded from file: {data}")- else:- # 尝试解析为 JSON 字符串- data = json.loads(data)- logger.info("Loaded from JSON string")-- # 恢复为 Schema 对象- if isinstance(data, dict):- for item_id, item_data in data.items():- item = self.schema_class.model_validate(item_data)- self._items[item_id] = item- self._updated_at[item_id] = datetime.now()-- logger.info(f"Loaded {len(self._items)} items")+ # 加载 items+ items_data = data.get("items", [])+ self._items = [self.schema_class.model_validate(item) for item in items_data]++ # 重建哈希集+ self._item_hashes = {self._compute_hash(item) for item in self._items}++ # 更新元数据+ if "metadata" in data:+ self.metadata.update(data["metadata"])+ self.metadata["updated_at"] = datetime.now()++ # 标记索引需要重建+ self._index_dirty = True++ logger.info(f"Loaded {len(self._items)} knowledge items successfully")- except Exception as e:- logger.error(f"Error loading: {e}")- raise+ def clear(self):+ """清空所有知识"""+ self._data = self._init_data()+ self._items.clear()+ self._item_hashes.clear()+ self.metadata["updated_at"] = datetime.now()+ self.metadata["total_extractions"] = 0+ self.metadata["total_chunks_processed"] = 0++ # 清空向量索引+ self._index = None+ self._index_dirty = True++ logger.info("Cleared all knowledge")diff --git a/src/knowledge/graph/base.py b/src/knowledge/graph/base.pyindex 260296b..1a27d1e 100644--- a/src/knowledge/graph/base.py+++ b/src/knowledge/graph/base.py@@ -59,13 +59,13 @@ class SimpleGraph(BaseKnowledge[GraphSchema]):# 4. 【直接在这里做合并逻辑】# 简单的去重合并- existing_node_ids = {n.id for n in self.data.nodes}+ existing_node_ids = {n.id for n in self._data.nodes}for node in parsed_data.nodes:if node.id not in existing_node_ids:- self.data.nodes.append(node)- self.data.edges.extend(parsed_data.edges)+ self._data.nodes.append(node)+ self._data.edges.extend(parsed_data.edges)def dump(self, format="json"):if format == "json":- return self.data.model_dump_json()- return self.data.model_dump()+ return self._data.model_dump_json()+ return self._data.model_dump()