Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

添加知识向量搜索功能示例,重构知识类以支持向量化提取和存储

yifanfeng97f851e1447a59
diff --git a/examples/knowledge_vector_search_demo.py b/examples/knowledge_vector_search_demo.py
new file mode 100644
index 0000000..dddca00
--- /dev/null
+++ b/examples/knowledge_vector_search_demo.py
@@ -0,0 +1,125 @@
+"""
+GenericKnowledge 向量搜索功能演示
+
+展示如何使用新的向量索引功能进行语义搜索。
+"""
+
+from pydantic import BaseModel, Field
+from typing import List
+
+
+# ==================== 1. 定义 Schema ====================
+
+class Article(BaseModel):
+ """文章 Schema"""
+ title: str = Field(description="文章标题")
+ author: str = Field(description="作者")
+ content: str = Field(description="文章内容摘要")
+ tags: List[str] = Field(default_factory=list, description="标签")
+
+
+# ==================== 2. 初始化 Knowledge ====================
+
+async def main():
+ from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+ from src.knowledge.generic import GenericKnowledge
+
+ # 初始化
+ knowledge = GenericKnowledge(
+ llm_client=ChatOpenAI(model="gpt-4"),
+ embedder=OpenAIEmbeddings(),
+ schema_class=Article,
+ prompt="Extract article information:",
+ max_workers=5,
+ show_progress=True
+ )
+
+ # ==================== 3. 提取知识 ====================
+
+ text = """
+ 人工智能的发展历程
+ 作者:张三
+
+ 人工智能(AI)从1956年达特茅斯会议诞生至今,经历了多次起伏。
+ 深度学习的突破使得AI在图像识别、自然语言处理等领域取得巨大进展。
+
+ 大语言模型的崛起
+ 作者:李四
+
+ 2023年,ChatGPT的发布标志着大语言模型时代的到来。
+ 这些模型能够理解和生成人类语言,应用范围广泛。
+ """
+
+ stats = knowledge.extract(text)
+ print(f"✅ 提取完成:{stats}")
+ print(f"📊 共提取 {knowledge.size()} 篇文章\n")
+
+ # ==================== 4. 向量搜索 ====================
+
+ print("🔍 开始语义搜索...")
+
+ # 搜索 1:关于深度学习
+ results = knowledge.search("深度学习和神经网络", top_k=2)
+ print("\n查询:'深度学习和神经网络'")
+ for i, article in enumerate(results, 1):
+ print(f" {i}. {article.title} - {article.author}")
+
+ # 搜索 2:关于 ChatGPT
+ results = knowledge.search("ChatGPT 语言模型", top_k=2)
+ print("\n查询:'ChatGPT 语言模型'")
+ for i, article in enumerate(results, 1):
+ print(f" {i}. {article.title} - {article.author}")
+
+ # ==================== 5. 持久化 ====================
+
+ # 保存
+ knowledge.dump("knowledge_data.json")
+ print("\n💾 已保存到 knowledge_data.json")
+
+ # 清空并重新加载
+ knowledge.clear()
+ print(f"🗑️ 清空后数量:{knowledge.size()}")
+
+ knowledge.load("knowledge_data.json")
+ print(f"📥 加载后数量:{knowledge.size()}")
+
+ # 加载后搜索仍然有效(索引会自动重建)
+ results = knowledge.search("AI 发展", top_k=1)
+ print(f"\n🔍 加载后搜索:找到 {len(results)} 条结果")
+
+
+# ==================== 6. 架构说明 ====================
+
+"""
+## 📐 架构设计
+
+### BaseKnowledge(模板方法模式)
+- `_vector_store`: 向量数据库实例(FAISS)
+- `_index_dirty`: 脏标记,追踪数据变化
+- `build_index()`: 模板方法,统一索引构建流程
+- `_data_to_documents()`: 抽象钩子,子类实现转换逻辑
+
+### GenericKnowledge(具体实现)
+1. **_data_to_documents()**: 将 self._items 转为 Document 列表
+2. **merge()**: 添加数据后设置 `self._index_dirty = True`
+3. **search()**:
+ - 调用 `build_index()` 确保索引最新
+ - 使用 `_vector_store.similarity_search()`
+ - 从 metadata 恢复原始对象
+4. **clear()**: 清空数据并重置索引
+
+### 工作流程
+```
+提取 → merge (设置 dirty=True) → search (自动 build_index) → 返回结果
+```
+
+### 优势
+✅ 延迟索引:只在需要时构建
+✅ 自动重建:数据变化后搜索时自动更新
+✅ 可插拔后端:FAISS/Qdrant/Chroma 等
+✅ 持久化友好:load 后自动标记 dirty
+"""
+
+if __name__ == "__main__":
+ import asyncio
+ asyncio.run(main())
diff --git a/src/knowledge/base.py b/src/knowledge/base.py
index f2a4579..81cbb28 100644
--- a/src/knowledge/base.py
+++ b/src/knowledge/base.py
@@ -1,329 +1,226 @@
-import asyncio
from abc import ABC, abstractmethod
-from typing import TypeVar, Generic, Any, Dict, Optional, Type, List, Callable
+from pathlib import Path
+from typing import TypeVar, Generic, Any, Dict, Type, List
from datetime import datetime
from pydantic import BaseModel
from langchain_core.language_models.chat_models import BaseChatModel
from langchain_core.embeddings import Embeddings
-try:
- from src.config import logger
-except ImportError:
- import logging
- logger = logging.getLogger(__name__)
-
T = TypeVar("T", bound=BaseModel)
-# ===================== 提取器 (Extractor) =====================
+# ===================== 知识类 (Knowledge) =====================
-class BaseExtractor(ABC, Generic[T]):
+class BaseKnowledge(ABC, Generic[T]):
"""
- 矿工 - 无状态提取器基类。
-
- 职责:从文本中提取结构化知识,不负责存储或融合。
- """
-
- def __init__(self, llm_client: BaseChatModel, schema_class: Type[T]):
- """
- :param llm_client: LangChain 的 ChatModel 实例
- :param schema_class: 期望的返回 Schema 类(必须继承 BaseModel)
- """
- self.llm_client = llm_client
- self.schema_class = schema_class
- @abstractmethod
- async def aparse(self, text: str, **kwargs) -> T:
- """
- 异步解析文本,返回单个知识对象。
-
- :param text: 输入文本
- :return: 解析后的知识对象
- """
- pass
-
- def parse(self, text: str, **kwargs) -> T:
- """
- 同步解析(子类可覆盖为真正的同步实现)。
-
- 默认行为:需要在异步上下文中调用 aparse。
- """
- try:
- loop = asyncio.get_event_loop()
- if loop.is_running():
- raise RuntimeError("Cannot use parse() in async context. Use aparse() instead.")
- return loop.run_until_complete(self.aparse(text, **kwargs))
- except RuntimeError:
- raise
-
- async def batch_parse(self, texts: List[str], **kwargs) -> List[T]:
- """
- 批量解析多个文本。
-
- :param texts: 文本列表
- :return: 解析结果列表
- """
- tasks = [self.aparse(text, **kwargs) for text in texts]
- return await asyncio.gather(*tasks)
-
- async def batch_parse_concurrent(
- self,
- texts: List[str],
- max_concurrent: int = 5,
- **kwargs
- ) -> List[T]:
- """
- 带并发限制的批量解析。
-
- :param texts: 文本列表
- :param max_concurrent: 最大并发数
- :return: 解析结果列表
- """
- from asyncio import Semaphore
-
- semaphore = Semaphore(max_concurrent)
-
- async def limited_parse(text: str) -> T:
- async with semaphore:
- return await self.aparse(text, **kwargs)
-
- tasks = [limited_parse(text) for text in texts]
- return await asyncio.gather(*tasks)
-
-
-
-
-
-# ===================== 集合/容器 (Collection) =====================
-
-
-class BaseCollection(ABC, Generic[T]):
- """
- 仓库 - 有状态集合基类。
-
- 职责:存储知识点、融合去重、演化优化、序列化。
+ 统一的知识基类 - 集提取、存储、聚合、演化于一体。
+
+ 职责:
+ 1. 从文本中提取结构化知识(extract)
+ 2. 自动处理长文本分块
+ 3. 存储和聚合提取的知识
+ 4. 演化优化内部知识
+ 5. 序列化与反序列化
"""
def __init__(
self,
llm_client: BaseChatModel,
embedder: Embeddings,
- schema_class: Type[T],
- storage: str = "memory",
+ *,
+ prompt: str = "",
+ chunk_size: int = 2000,
+ chunk_overlap: int = 200,
+ max_workers: int = 10,
+ show_progress: bool = True,
**kwargs,
):
"""
- :param llm_client: LLM 实例(用于演化等操作)
- :param embedder: 向量化器实例(用于语义搜索等)
- :param schema_class: 容器持有的知识 Schema 类
- :param storage: 存储后端类型("memory"、"postgres" 等)
+ 初始化知识对象。
+
+ :param llm_client: LLM 客户端(用于提取和演化)
+ :param embedder: 向量化器(用于语义搜索和相似度计算)
+ :param prompt: 用户自定义的提示词(用于 extract)
+ :param chunk_size: 长文本分块大小
+ :param chunk_overlap: 分块重叠大小
+ :param max_workers: 最大并发数(用于并发提取)
+ :param show_progress: 是否显示进度信息
"""
self.llm_client = llm_client
self.embedder = embedder
- self.schema_class = schema_class
- self.storage = storage
- self.metadata: Dict[str, Any] = {}
-
- # ================= 存储层 =================
+ self.chunk_size = chunk_size
+ self.chunk_overlap = chunk_overlap
+ self.prompt = prompt
+ self.max_workers = max_workers
+ self.show_progress = show_progress
+
+ # 向量存储相关(使用 FAISS)
+ self._index = None
+ self._index_dirty: bool = True # 标记索引是否需要重建
+
+ # 内部状态:存储提取的知识
+ self._data: T = self._init_data()
+ self.metadata: Dict[str, Any] = {
+ "created_at": datetime.now(),
+ "updated_at": datetime.now(),
+ "total_extractions": 0,
+ "total_chunks_processed": 0,
+ }
- @abstractmethod
- async def add(self, item_id: str, item: T, **kwargs):
+ @property
+ def schema_class(self) -> Type[T]:
"""
- 添加知识点。
-
- :param item_id: 唯一标识符
- :param item: 知识对象
+ 返回当前知识类使用的 Schema 类。
+ 子类可覆盖此属性,或在泛型参数中指定。
"""
- pass
+ # 默认实现:从泛型参数推导(仅当子类未覆盖时)
+ raise NotImplementedError("schema_class must be implemented by subclass")
@abstractmethod
- async def remove(self, item_id: str, **kwargs) -> bool:
+ def _init_data(self) -> T:
"""
- 删除知识点。
-
- :return: 是否成功删除
+ 初始化内部数据结构。
+
+ :return: 初始化的知识容器
"""
- pass
+ return self.schema_class()
+
+ # ==================== 向量索引管理(模板方法模式)====================
@abstractmethod
- async def update(self, item_id: str, item: T, **kwargs):
+ def build_index(self):
"""
- 更新知识点。
-
- :param item_id: 知识点 ID
- :param item: 新的知识对象
+ 构建或重建向量索引。
+
+ 子类必须实现此方法,定义如何构建向量索引。
+ 使用 FAISS 作为向量存储后端。
"""
pass
@abstractmethod
- def get(self, item_id: str) -> Optional[T]:
- """获取单个知识点"""
+ def clear(self):
+ """清空所有知识。"""
pass
- @abstractmethod
- def list_all(self) -> List[T]:
- """列出所有知识点"""
- pass
+ # ==================== 核心方法:extract ====================
@abstractmethod
- def search(self, filter_fn: Callable[[T], bool]) -> List[T]:
- """
- 按条件查询知识点。
-
- :param filter_fn: 过滤函数
+ def extract(self, text: str, **kwargs) -> Dict[str, Any]:
"""
- pass
+ 主提取方法 - 自动处理长文本分块、提取、聚合。
- @abstractmethod
- def size(self) -> int:
- """返回容器中知识点的数量"""
+ 流程:
+ 1. 判断文本长度,决定是否分块
+ 2. 对每个块调用 _extract_chunk() 进行提取
+ 3. 调用 merge() 将提取结果聚合到 self._data
+ 4. 返回提取统计信息
+
+ :param text: 输入文本(可以是短文本或长文本)
+ :return: 提取统计信息 {"chunks_processed": 3, "items_extracted": 15, ...}
+ """
pass
- # ================= 融合层 =================
+ # ==================== 存储与聚合 ====================
@abstractmethod
- async def aggregate(self, items: List[T], **kwargs) -> Dict[str, Any]:
+ def merge(self, items: List[T], **kwargs) -> Dict[str, Any]:
"""
- 融合多个知识点。
-
- 职责:清洗、去重、合并。
-
- :param items: 新提取的知识点列表
- :return: 融合统计信息(去重数、合并数等)
+ 将多个提取结果合并到内部状态 self._data。
+
+ 职责:去重、合并、冲突解决。
+ 子类必须实现此方法。
+
+ :param items: 从各个块提取的知识列表
+ :return: 合并统计信息 {"items_added": 10, "duplicates_removed": 2, ...}
"""
pass
- async def fuse_batch(self, new_items: List[T], iteration: int = 0) -> Dict[str, Any]:
- """
- 融合一批新知识点到容器中。
-
- :param new_items: 新提取的知识点
- :param iteration: 迭代号
- :return: 融合统计信息
+ # ==================== 查询接口 ====================
+
+ @property
+ def data(self) -> T:
"""
- start_time = datetime.now()
+ 获取所有存储的知识(只读)。
- try:
- # 应用融合策略
- stats = await self.aggregate(new_items, iteration=iteration)
+ :return: 内部知识数据
+ """
+ return self._data
- stats["timestamp"] = start_time.isoformat()
- stats["duration_ms"] = (datetime.now() - start_time).total_seconds() * 1000
- stats["iteration"] = iteration
+ @abstractmethod
+ def search(self, query: str, top_k: int = 10, **kwargs) -> List[Any]:
+ """
+ 语义搜索知识。
- logger.info(f"Fused batch: {stats}")
- return stats
+ 默认实现:
+ 1. 确保索引已构建(调用 build_index)
+ 2. 使用 vector_store.similarity_search 进行检索
+ 3. 从 Document.metadata 恢复原始数据结构
- except Exception as e:
- logger.error(f"Error during fuse_batch: {e}")
- raise
+ 子类可覆盖此方法实现自定义搜索逻辑。
- # ================= 演化层 =================
+ :param query: 查询字符串
+ :param top_k: 返回结果数
+ :return: 相关知识列表
+ """
+ pass
@abstractmethod
- async def evolve(self, **kwargs) -> Dict[str, Any]:
+ def size(self) -> int:
"""
- 演化内部知识。
-
- 职责:推理新关系、剪枝低置信度、聚类等高级操作。
-
- :return: 演化统计信息
+ 返回知识点数量。
+ 子类必须实现此方法。
+
+ :return: 知识点数量
"""
pass
- # ================= 高级流程 =================
+ # ==================== 演化 ====================
- async def iterative_fuse_and_parse(
- self,
- extractor: "BaseExtractor[T]",
- texts: List[str],
- max_iterations: int = 3,
- convergence_threshold: float = 0.95,
- ) -> Dict[str, Any]:
- """
- 迭代提取与融合流程。
-
- 流程:
- 1. 第一次提取:从文本中提取所有知识点
- 2. 融合去重
- 3. 演化优化
- 4. 重复直到收敛或达到最大迭代数
-
- :param extractor: 知识提取器
- :param texts: 输入文本列表
- :param max_iterations: 最大迭代数
- :param convergence_threshold: 收敛阈值
- :return: 迭代结果摘要
+ @abstractmethod
+ def evolve(self, **kwargs) -> T:
"""
- iteration = 0
- previous_size = 0
- fusion_stats = []
-
- while iteration < max_iterations:
- logger.info(f"Iteration {iteration}: extracting...")
-
- # 1. 提取新知识
- extracted = await extractor.batch_parse(texts)
-
- # 2. 融合
- logger.info(f"Iteration {iteration}: fusing...")
- fuse_result = await self.fuse_batch(extracted, iteration)
- fusion_stats.append(fuse_result)
-
- # 3. 演化
- logger.info(f"Iteration {iteration}: evolving...")
- evolve_result = await self.evolve(iteration=iteration)
-
- # 4. 检查收敛
- current_size = self.size()
- logger.info(f"Iteration {iteration}: size={current_size}")
-
- if previous_size > 0:
- growth_rate = (current_size - previous_size) / previous_size
- if growth_rate < (1 - convergence_threshold):
- logger.info(
- f"Converged at iteration {iteration} (growth_rate={growth_rate})"
- )
- break
-
- previous_size = current_size
- iteration += 1
-
- await asyncio.sleep(0.1)
-
- return {
- "final_iteration": iteration,
- "total_items": self.size(),
- "fusion_history": fusion_stats,
- }
+ 演化内部知识。
+ 子类必须实现此方法。
- # ================= 序列化 =================
+ 职责:
+ - 推理隐含关系
+ - 剪枝低置信度节点
+ - 聚类优化
+ - 知识补全
- @abstractmethod
- def dump(self, format: str = "json") -> Any:
- """
- 导出容器内的知识。
-
- :param format: 格式 ("json", "dict" 等)
+ :return: 新的内部知识数据
"""
pass
+ # ==================== 序列化 ====================
+
@abstractmethod
- def load(self, data: Any, **kwargs):
- """
- 从外部数据恢复知识。
-
- :param data: 外部数据 (dict, json string 或文件路径)
+ def dump(self, folder_path: str | Path) -> Any:
"""
- pass
+ 导出知识到指定文件夹。
+ 子类必须实现此方法。
- def clear(self):
- """清空所有知识点(可选实现)"""
- pass
+ 需要保存到文件夹内:
+ 1. 结构化数据 (self._data) - 保存为 JSON 文件
+ 2. 向量索引 (self._index) - FAISS 索引文件
+ :param folder_path: 目标文件夹路径
+ :return: 序列化后的数据
+ """
+ pass
-# ===================== 通用实现 =====================
-
+ @abstractmethod
+ def load(self, folder_path: str | Path, **kwargs):
+ """
+ 从文件夹加载知识。
+ 子类必须实现此方法。
+ 需要从文件夹加载:
+ 1. 结构化数据 (self._data) - 从 JSON 文件加载
+ 2. 向量索引 (self._index) - 从 FAISS 索引文件加载
+ :param folder_path: 文件夹路径
+ """
+ pass
diff --git a/src/knowledge/generic.py b/src/knowledge/generic.py
index 75af59b..9a2029f 100644
--- a/src/knowledge/generic.py
+++ b/src/knowledge/generic.py
@@ -1,18 +1,20 @@
"""
通用 Generic 实现。
-包含基于 BaseExtractor 和 BaseCollection 的具体实现。
+包含基于 BaseKnowledge 的具体实现。
"""
-import os
import json
-from typing import List, Dict, Any, Callable, Optional, Type
+import hashlib
+from typing import List, Dict, Any, Optional, Type
from datetime import datetime
-from pydantic import BaseModel, ValidationError
+from pydantic import ValidationError
from langchain_core.language_models.chat_models import BaseChatModel
from langchain_core.embeddings import Embeddings
+from langchain_core.documents import Document
+from langchain_core.vectorstores import VectorStore
-from .base import BaseExtractor, BaseCollection, T
+from .base import BaseKnowledge, T
try:
from src.config import logger
@@ -21,36 +23,14 @@ except ImportError:
logger = logging.getLogger(__name__)
-class GenericExtractor(BaseExtractor[T]):
+class GenericKnowledge(BaseKnowledge[T]):
"""
- 通用提取器。使用 LLM 的 with_structured_output 实现提取。
- """
-
- async def aparse(self, text: str, **kwargs) -> T:
- """
- 使用 with_structured_output 异步解析。
- """
- try:
- # 创建支持结构化输出的 LLM
- structured_llm = self.llm_client.with_structured_output(self.schema_class)
-
- # 调用 LLM
- result: T = await structured_llm.ainvoke(text)
-
- logger.info(f"Successfully parsed: {type(result).__name__}")
- return result
-
- except ValidationError as e:
- logger.error(f"Schema validation failed: {e}")
- raise
- except Exception as e:
- logger.error(f"Error during parsing: {e}")
- raise
-
-
-class GenericCollection(BaseCollection[T]):
- """
- 通用集合实现。支持任何 Pydantic BaseModel Schema。
+ 通用知识实现 - 适用于任何 Pydantic Schema。
+
+ 特点:
+ - 使用 LLM 的 with_structured_output 进行提取
+ - 基于 JSON 哈希进行去重
+ - 支持向量化语义搜索
"""
def __init__(
@@ -58,147 +38,370 @@ class GenericCollection(BaseCollection[T]):
llm_client: BaseChatModel,
embedder: Embeddings,
schema_class: Type[T],
- storage: str = "memory",
+ deduplication_strategy: str = "hash",
**kwargs,
):
"""
:param llm_client: LLM 实例
:param embedder: 向量化器实例
:param schema_class: Schema 类
- :param storage: 存储类型
- """
- super().__init__(llm_client, embedder, schema_class, storage, **kwargs)
- self._items: Dict[str, T] = {}
- self._updated_at: Dict[str, datetime] = {}
-
- # ================= 存储层实现 =================
-
- async def add(self, item_id: str, item: T, **kwargs):
- """添加知识点"""
- self._items[item_id] = item
- self._updated_at[item_id] = datetime.now()
- logger.info(f"Added item: {item_id}")
-
- async def remove(self, item_id: str, **kwargs) -> bool:
- """删除知识点"""
- if item_id in self._items:
- del self._items[item_id]
- del self._updated_at[item_id]
- logger.info(f"Removed item: {item_id}")
- return True
- return False
-
- async def update(self, item_id: str, item: T, **kwargs):
- """更新知识点"""
- if item_id in self._items:
- self._items[item_id] = item
- self._updated_at[item_id] = datetime.now()
- logger.info(f"Updated item: {item_id}")
- else:
- await self.add(item_id, item, **kwargs)
-
- def get(self, item_id: str) -> Optional[T]:
- """获取单个知识点"""
- return self._items.get(item_id)
+ :param deduplication_strategy: 去重策略("hash" 或 "semantic")
+ :param kwargs: 其他参数(prompt, max_workers, show_progress 等)
+ """
+ super().__init__(llm_client, embedder, **kwargs)
+ self._schema_class = schema_class
+ self.deduplication_strategy = deduplication_strategy
+ self._items: List[T] = [] # 内部存储列表
+ self._item_hashes: set = set() # 用于基于哈希的去重
+
+ @property
+ def schema_class(self) -> Type[T]:
+ """返回当前使用的 Schema 类"""
+ return self._schema_class
+
+ def _init_data(self) -> T:
+ """初始化为 Schema 类的空实例"""
+ try:
+ return self.schema_class()
+ except Exception:
+ return self.schema_class.model_construct()
- def list_all(self) -> List[T]:
- """列出所有知识点"""
- return list(self._items.values())
+ def build_index(self):
+ """
+ 构建或重建向量索引(使用 FAISS)。
+ """
+ if not self._index_dirty and self._index is not None:
+ return # 索引已是最新
+
+ # 1. 将 items 转为 Document 列表
+ documents = []
+ for i, item in enumerate(self._items):
+ documents.append(Document(
+ page_content=item.model_dump_json(),
+ metadata={
+ "index": i,
+ "hash": self._compute_hash(item),
+ "raw": item.model_dump(),
+ }
+ ))
+
+ if not documents:
+ logger.warning("No documents to index")
+ return
+
+ # 2. 使用 FAISS 构建向量存储
+ try:
+ from langchain_community.vectorstores import FAISS
+ self._index = FAISS.from_documents(
+ documents=documents,
+ embedding=self.embedder
+ )
+ self._index_dirty = False
+ logger.info(f"Built FAISS index with {len(documents)} documents")
+ except ImportError:
+ logger.error("FAISS not available. Install with: pip install faiss-cpu")
+ raise
- def search(self, filter_fn: Callable[[T], bool]) -> List[T]:
- """按条件查询"""
- return [item for item in self._items.values() if filter_fn(item)]
+ def extract(self, text: str, **kwargs) -> Dict[str, Any]:
+ """主提取方法 - 自动处理长文本分块、提取、聚合"""
+ import asyncio
+ from datetime import datetime
+ start_time = datetime.now()
+
+ # 1. 分块
+ chunks = self._chunk_text(text) if len(text) > self.chunk_size else [text]
+ if self.show_progress:
+ logger.info(f"Processing {len(chunks)} chunk(s) with max_workers={self.max_workers}...")
+
+ # 2. 并发提取
+ async def extract_all():
+ from asyncio import Semaphore
+ semaphore = Semaphore(self.max_workers)
+
+ async def limited_extract(i: int, chunk: str) -> Optional[T]:
+ async with semaphore:
+ try:
+ if self.show_progress:
+ logger.info(f"Extracting chunk {i + 1}/{len(chunks)}...")
+ return await self._extract_chunk(chunk, **kwargs)
+ except Exception as e:
+ logger.error(f"Error extracting chunk {i + 1}: {e}")
+ return None
+
+ tasks = [limited_extract(i, chunk) for i, chunk in enumerate(chunks)]
+ return await asyncio.gather(*tasks)
+
+ # 运行异步提取
+ try:
+ loop = asyncio.get_event_loop()
+ if loop.is_running():
+ # 如果已在异步环境中,尝试使用 nest_asyncio(如果可用)
+ try:
+ import nest_asyncio
+ nest_asyncio.apply()
+ results = loop.run_until_complete(extract_all())
+ except ImportError:
+ logger.warning("nest_asyncio not available, creating new event loop")
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+ results = loop.run_until_complete(extract_all())
+ else:
+ results = loop.run_until_complete(extract_all())
+ except RuntimeError:
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+ results = loop.run_until_complete(extract_all())
+
+ chunk_results = [r for r in results if r is not None]
- def size(self) -> int:
- """返回知识点数量"""
- return len(self._items)
+ # 3. 合并
+ logger.info("Merging extracted knowledge...")
+ aggregate_stats = self.merge(chunk_results, **kwargs)
- def clear(self):
- """清空所有知识点"""
- self._items.clear()
- self._updated_at.clear()
- logger.info("Cleared all items")
+ # 4. 更新元数据
+ self.metadata["updated_at"] = datetime.now()
+ self.metadata["total_extractions"] += 1
+ self.metadata["total_chunks_processed"] += len(chunks)
- # ================= 融合层实现 =================
+ # 5. 返回统计
+ duration = (datetime.now() - start_time).total_seconds()
+ return {
+ "success": True,
+ "chunks_processed": len(chunks),
+ "chunks_succeeded": len(chunk_results),
+ "aggregate_stats": aggregate_stats,
+ "duration_seconds": duration,
+ "timestamp": start_time.isoformat(),
+ }
- async def aggregate(self, items: List[T], **kwargs) -> Dict[str, Any]:
+ def _chunk_text(self, text: str) -> List[str]:
+ """将长文本分块"""
+ try:
+ from langchain_text_splitters import RecursiveCharacterTextSplitter
+ splitter = RecursiveCharacterTextSplitter(
+ chunk_size=self.chunk_size,
+ chunk_overlap=self.chunk_overlap,
+ separators=["\n\n", "\n", "。", "!", "?", ". ", "! ", "? ", " ", ""],
+ )
+ chunks = splitter.split_text(text)
+ logger.info(f"Split text into {len(chunks)} chunks")
+ return chunks
+ except ImportError:
+ logger.warning("langchain_text_splitters not available, using simple chunking")
+ chunks = []
+ for i in range(0, len(text), self.chunk_size - self.chunk_overlap):
+ chunks.append(text[i:i + self.chunk_size])
+ return chunks
+
+ async def _extract_chunk(self, chunk: str, **kwargs) -> T:
"""
- 默认融合实现:简单去重合并。
+ 使用 LLM 的 with_structured_output 提取单个块。
- 子类可覆盖此方法实现更复杂的融合逻辑。
+ :param chunk: 文本块
+ :return: 提取的知识对象
"""
- logger.info(f"Aggregating {len(items)} items...")
-
- # 简单的聚合:直接添加到容器
- for idx, item in enumerate(items):
- item_id = f"item_{len(self._items)}_{idx}"
- await self.add(item_id, item)
+ try:
+ structured_llm = self.llm_client.with_structured_output(self.schema_class)
+
+ # 如果有自定义 prompt,则在消息前加上
+ if self.prompt:
+ full_text = f"{self.prompt}\n\n{chunk}"
+ else:
+ full_text = chunk
+
+ result: T = await structured_llm.ainvoke(full_text)
+ logger.info(f"Extracted from chunk: {type(result).__name__}")
+ return result
+ except ValidationError as e:
+ logger.error(f"Schema validation failed: {e}")
+ raise
+ except Exception as e:
+ logger.error(f"Error during extraction: {e}")
+ raise
+ def merge(self, items: List[T], **kwargs) -> Dict[str, Any]:
+ """
+ 合并提取结果到内部列表。
+
+ :param items: 提取的知识列表
+ :return: 合并统计
+ """
+ added = 0
+ duplicates = 0
+
+ for item in items:
+ # 计算哈希用于去重
+ item_hash = self._compute_hash(item)
+
+ if item_hash not in self._item_hashes:
+ self._items.append(item)
+ self._item_hashes.add(item_hash)
+ added += 1
+ logger.debug(f"Added item: {item_hash[:8]}...")
+ else:
+ duplicates += 1
+ logger.debug(f"Skipped duplicate: {item_hash[:8]}...")
+
+ # 更新 self.data(如果 Schema 有 items 字段)
+ if hasattr(self._data, "items") and hasattr(self._data.items, "append"):
+ # 如果 data.items 是列表,更新它
+ self._data.items = self._items
+
+ # 标记索引需要重建
+ self._index_dirty = True
+
return {
- "input_count": len(items),
- "output_count": len(items),
- "deduplicated": 0,
+ "items_added": added,
+ "duplicates_removed": duplicates,
+ "total_items": len(self._items),
}
- # ================= 演化层实现 =================
+ def _compute_hash(self, item: T) -> str:
+ """计算知识对象的哈希值"""
+ item_json = item.model_dump_json(sort_keys=True)
+ return hashlib.sha256(item_json.encode()).hexdigest()
- async def evolve(self, **kwargs) -> Dict[str, Any]:
+ def search(self, query: str, top_k: int = 10, **kwargs) -> List[T]:
+ """
+ 基于向量索引的语义搜索。
+
+ :param query: 查询字符串
+ :param top_k: 返回结果数
+ :return: 最相关的知识列表
+ """
+ if not self._items:
+ logger.warning("No items to search")
+ return []
+
+ try:
+ # 1. 确保索引已构建
+ self.build_index()
+
+ if self._index is None:
+ logger.warning("Vector store not available, falling back to text matching")
+ raise Exception("Vector store not initialized")
+
+ # 2. 使用向量存储进行语义搜索
+ docs = self._index.similarity_search(query, k=top_k)
+
+ # 3. 从 metadata 恢复原始对象
+ results = []
+ for doc in docs:
+ try:
+ item = self.schema_class.model_validate(doc.metadata["raw"])
+ results.append(item)
+ except Exception as e:
+ logger.warning(f"Failed to restore item from metadata: {e}")
+ continue
+
+ logger.info(f"Found {len(results)} results for query: {query[:50]}...")
+ return results
+
+ except Exception as e:
+ logger.error(f"Error during vector search: {e}, falling back to text matching")
+ # 降级到简单的文本匹配
+ return [
+ item for item in self._items
+ if query.lower() in item.model_dump_json().lower()
+ ][:top_k]
+
+ def size(self) -> int:
+ """返回知识点数量"""
+ return len(self._items)
+
+ async def evolve(self, **kwargs) -> T:
"""
默认演化实现:空操作。
子类可覆盖此方法实现知识演化、推理、剪枝等逻辑。
+
+ :return: 内部知识数据
"""
logger.debug("Default evolve: no operation")
- return {"evolved": False, "message": "Default implementation"}
+ return self._data
- # ================= 序列化实现 =================
+ def get_all_items(self) -> List[T]:
+ """获取所有知识项列表"""
+ return self._items.copy()
- def dump(self, format: str = "json") -> Any:
- """导出容器内的知识"""
- if not self._items:
- logger.warning("No items to dump")
- return None
+ def dump(self, filepath: str) -> Any:
+ """
+ 导出知识为 JSON 文件。
- try:
- if format.lower() == "json":
- data = {
- k: v.model_dump() for k, v in self._items.items()
- }
- return json.dumps(data, indent=2, ensure_ascii=False)
+ :param filepath: 文件路径
+ :return: 序列化后的数据(JSON 字符串)
+ """
+ from pathlib import Path
+
+ data = {
+ "schema_class": self._schema_class.__name__,
+ "items": [item.model_dump() for item in self._items],
+ "metadata": {k: str(v) if isinstance(v, datetime) else v for k, v in self.metadata.items()},
+ }
+ json_str = json.dumps(data, indent=2, ensure_ascii=False, default=str)
- elif format.lower() == "dict":
- return {k: v.model_dump() for k, v in self._items.items()}
+ filepath = Path(filepath)
+ filepath.parent.mkdir(parents=True, exist_ok=True)
+
+ with open(filepath, "w", encoding="utf-8") as f:
+ f.write(json_str)
+
+ logger.info(f"Dumped {len(self._items)} items to {filepath}")
+ return json_str
- else:
- raise ValueError(f"Unsupported format: {format}")
+ def load(self, path_or_json: str | dict, **kwargs):
+ """
+ 加载知识。
- except Exception as e:
- logger.error(f"Error dumping: {e}")
- raise
+ :param path_or_json: JSON 文件路径或 JSON 字符串或字典
+ """
+ from pathlib import Path
+
+ # 解析输入
+ if isinstance(path_or_json, dict):
+ data = path_or_json
+ logger.info("Loading from dictionary")
+ elif isinstance(path_or_json, str):
+ path = Path(path_or_json)
+ if path.is_file():
+ with open(path, "r", encoding="utf-8") as f:
+ data = json.load(f)
+ logger.info(f"Loaded from file: {path}")
+ else:
+ # 尝试作为 JSON 字符串解析
+ data = json.loads(path_or_json)
+ logger.info("Loaded from JSON string")
+ else:
+ raise ValueError(f"Unsupported data type: {type(path_or_json)}")
- def load(self, data: Any, **kwargs):
- """从外部数据恢复知识"""
- try:
- if isinstance(data, str):
- # 检查是否是文件路径
- if os.path.isfile(data):
- with open(data, "r", encoding="utf-8") as f:
- data = json.load(f)
- logger.info(f"Loaded from file: {data}")
- else:
- # 尝试解析为 JSON 字符串
- data = json.loads(data)
- logger.info("Loaded from JSON string")
-
- # 恢复为 Schema 对象
- if isinstance(data, dict):
- for item_id, item_data in data.items():
- item = self.schema_class.model_validate(item_data)
- self._items[item_id] = item
- self._updated_at[item_id] = datetime.now()
-
- logger.info(f"Loaded {len(self._items)} items")
+ # 加载 items
+ items_data = data.get("items", [])
+ self._items = [self.schema_class.model_validate(item) for item in items_data]
+
+ # 重建哈希集
+ self._item_hashes = {self._compute_hash(item) for item in self._items}
+
+ # 更新元数据
+ if "metadata" in data:
+ self.metadata.update(data["metadata"])
+ self.metadata["updated_at"] = datetime.now()
+
+ # 标记索引需要重建
+ self._index_dirty = True
+
+ logger.info(f"Loaded {len(self._items)} knowledge items successfully")
- except Exception as e:
- logger.error(f"Error loading: {e}")
- raise
+ def clear(self):
+ """清空所有知识"""
+ self._data = self._init_data()
+ self._items.clear()
+ self._item_hashes.clear()
+ self.metadata["updated_at"] = datetime.now()
+ self.metadata["total_extractions"] = 0
+ self.metadata["total_chunks_processed"] = 0
+
+ # 清空向量索引
+ self._index = None
+ self._index_dirty = True
+
+ logger.info("Cleared all knowledge")
diff --git a/src/knowledge/graph/base.py b/src/knowledge/graph/base.py
index 260296b..1a27d1e 100644
--- a/src/knowledge/graph/base.py
+++ b/src/knowledge/graph/base.py
@@ -59,13 +59,13 @@ class SimpleGraph(BaseKnowledge[GraphSchema]):
# 4. 【直接在这里做合并逻辑】
# 简单的去重合并
- existing_node_ids = {n.id for n in self.data.nodes}
+ existing_node_ids = {n.id for n in self._data.nodes}
for node in parsed_data.nodes:
if node.id not in existing_node_ids:
- self.data.nodes.append(node)
- self.data.edges.extend(parsed_data.edges)
+ self._data.nodes.append(node)
+ self._data.edges.extend(parsed_data.edges)
def dump(self, format="json"):
if format == "json":
- return self.data.model_dump_json()
- return self.data.model_dump()
+ return self._data.model_dump_json()
+ return self._data.model_dump()