Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
重构知识提取模块,优化默认提示词和合并逻辑,添加实体知识提取实现
yifanfeng9726ea93f4dc12
diff --git a/examples/generic_knowledge_demo.py b/examples/generic_knowledge_demo.pyindex 9bc30cd..513d218 100644--- a/examples/generic_knowledge_demo.py+++ b/examples/generic_knowledge_demo.py@@ -98,7 +98,7 @@ def main():# 8. 保存知识print("\n[7] 保存知识到文件夹...")- save_path = "./.pytest_cache/article_knowledge"+ save_path = "./tmp/article_knowledge"knowledge.dump(save_path)print(f"✓ 知识已保存到: {save_path}")diff --git a/src/knowledge/base.py b/src/knowledge/base.pyindex a99f4c5..2c631c3 100644--- a/src/knowledge/base.py+++ b/src/knowledge/base.py@@ -54,22 +54,13 @@ class BaseKnowledge(ABC, Generic[T]):self._data_schema = data_schemaself.llm_client = llm_clientself.embedder = embedder+ self.prompt = prompt or self._default_prompt()self.chunk_size = chunk_sizeself.chunk_overlap = chunk_overlapself.max_workers = max_workersself.show_progress = show_progress# 初始化提示词模板和 LLM 链- if not prompt:- self.prompt =(- "You are an expert knowledge extraction assistant. "- "Your task is to carefully analyze the following text and extract structured information "- "according to the specified schema. Be precise, comprehensive, and faithful to the source text. "- "Extract all relevant details without adding information not present in the text.\n\n"- "### Source Text:\n"- )- else:- self.prompt = promptself.prompt_template = ChatPromptTemplate.from_template(f"{self.prompt}{{chunk_text}}")self.llm_with_schema = self.llm_client.with_structured_output(self._data_schema)self.llm_chain = self.prompt_template | self.llm_with_schema@@ -92,6 +83,17 @@ class BaseKnowledge(ABC, Generic[T]):"updated_at": datetime.now(),}+ @staticmethod+ def _default_prompt() -> str:+ """默认提取提示词"""+ return (+ "You are an expert knowledge extraction assistant. "+ "Your task is to carefully analyze the following text and extract structured information "+ "according to the specified schema. Be precise, comprehensive, and faithful to the source text. "+ "Extract all relevant details without adding information not present in the text.\n\n"+ "### Source Text:\n"+ )+# ==================== 数据管理 ====================@property@@ -120,7 +122,7 @@ class BaseKnowledge(ABC, Generic[T]):# ==================== 提取与聚合 ====================@abstractmethod- def extract(self, text: str) -> Dict[str, Any]:+ def extract(self, text: str) -> T:"""主提取方法 - 自动处理长文本分块、提取、聚合。@@ -128,29 +130,28 @@ class BaseKnowledge(ABC, Generic[T]):1. 判断文本长度,决定是否分块2. 对每个块调用 _extract_chunk() 进行提取3. 调用 merge() 将提取结果聚合到 self._data- 4. 返回提取统计信息+ 4. 返回提取到的结果:param text: 输入文本(可以是短文本或长文本)- :return: 提取统计信息 {"chunks_processed": 3, "items_extracted": 15, ...}+ :return: 提取到的知识数据"""pass@abstractmethod- def merge(self, data_list: List[T]) -> Dict[str, Any]:+ def merge(self, data_list: List[T]) -> T:"""将多个提取结果合并到内部状态 self._data。职责:去重、合并、冲突解决。- 子类必须实现此方法。+ 子类必须实现此方法.:param data_list: 从各个块提取的知识列表- :return: 合并统计信息 {"items_added": 10, "duplicates_removed": 2, ...}+ :return: 合并后的知识数据"""pass# ==================== 演化 ====================- @abstractmethoddef evolve(self, **kwargs) -> T:"""演化内部知识。diff --git a/src/knowledge/entity.py b/src/knowledge/entity.pynew file mode 100644index 0000000..fe913a1--- /dev/null+++ b/src/knowledge/entity.py@@ -0,0 +1,396 @@+"""+实体知识提取模块。++包含基于 GenericKnowledge 的实体提取实现。+"""++import json+from typing import List, Dict, Any, Type+from datetime import datetime+from pydantic import BaseModel, Field as PydanticField, create_model+from langchain_core.language_models.chat_models import BaseChatModel+from langchain_core.embeddings import Embeddings+from langchain_core.documents import Document+from langchain_core.prompts import ChatPromptTemplate++from .generic import GenericKnowledge+from .base import T++try:+ from src.config import logger+except ImportError:+ import logging++ logger = logging.getLogger(__name__)+++class BaseEntity(BaseModel):+ """+ 实体基类 - 所有实体必须包含的核心字段。++ 用户可继承此类添加自定义字段。+ """++ name: str = PydanticField(description="实体唯一名称", min_length=1)+ description: str = PydanticField(description="实体描述")++ class Config:+ """允许用户扩展字段"""++ extra = "allow" # 允许额外字段+ validate_assignment = True # 赋值时验证+++class EntityKnowledge(GenericKnowledge[T]):+ """+ 专门用于实体提取和管理的知识类。++ 特点:+ 1. 自动去重(基于 entity.name)+ 2. LLM 智能合并(同名实体的字段合并)+ 3. 每个实体作为独立文档建索引+ 4. 支持实体级别的搜索+ """++ def __init__(+ self,+ entity_schema: Type[BaseEntity],+ llm_client: BaseChatModel,+ embedder: Embeddings,+ *,+ prompt: str = "",+ **kwargs,+ ):+ """+ :param entity_schema: 用户自定义的实体类(必须继承 BaseEntity)+ :param llm_client: LLM 客户端+ :param embedder: 向量化器+ :param prompt: 自定义提示词+ """+ self._entity_schema = entity_schema+ entity_list_schema = self._create_entity_list_schema(entity_schema)++ super().__init__(+ data_schema=entity_list_schema,+ llm_client=llm_client,+ embedder=embedder,+ prompt=prompt or self._default_prompt(),+ **kwargs,+ )++ self._merge_chain = self._create_merge_chain()++ # 全局实体映射(主存储)+ self._entity_map: Dict[str, BaseEntity] = {}+ self._entity_map_dirty = True # 标记是否需要同步到 _data++ @staticmethod+ def _create_entity_list_schema(entity_class: Type[BaseEntity]) -> Type[BaseModel]:+ """使用 pydantic.create_model 动态创建 EntityList Schema"""+ return create_model(+ f"{entity_class.__name__}List",+ entities=(+ List[entity_class],+ PydanticField(default_factory=list, description="实体列表"),+ ),+ __base__=BaseModel,+ )++ @staticmethod+ def _default_prompt() -> str:+ """默认实体提取提示词"""+ return (+ "You are an expert entity extraction assistant. "+ "Carefully read the following text and extract all entities "+ "according to the specified schema. Be precise and comprehensive."+ "Extract all entities without adding information not present in the text.\n\n"+ "### Source Text:\n"+ )++ def _create_merge_chain(self):+ """创建 LLM 合并 chain,用于智能合并同名实体"""+ merge_template = """You are an expert at merging entity information.+Given two instances of the same entity, intelligently merge their fields into one.++Rules:+1. Keep the 'name' field unchanged+2. Merge 'description' by combining both descriptions (remove duplicates)+3. For other fields: prefer non-null values, if both exist choose the more complete/accurate one++Entity A:+{entity_a}++Entity B:+{entity_b}++Please merge these two entities intelligently and return the result."""++ merge_prompt = ChatPromptTemplate.from_template(merge_template)++ return merge_prompt | self.llm_client.with_structured_output(+ self._entity_schema+ )++ # ==================== 数据管理 ====================++ @property+ def data(self) -> T:+ """懒更新:只在访问时才同步 entity_map 到 _data"""+ if self._entity_map_dirty:+ self._data.entities = [+ entity.model_copy(deep=True)+ for entity in self._entity_map.values()+ ]+ self._entity_map_dirty = False+ return self._data++ def clear(self):+ """清空所有实体"""+ self._entity_map.clear()+ self._entity_map_dirty = True+ self._data = self.data_schema()+ self._index = None+ self._index_dirty = True+ self.metadata["updated_at"] = datetime.now()+ logger.info("Cleared all entities")++ # ==================== 提取与聚合 ====================++ def merge(self, data_list: List[T]) -> T:+ """+ 实体级别的智能合并(使用 LLM)。++ 策略:+ 1. 与已有的 entity_map 合并(累积效果)+ 2. 使用 name 作为唯一标识符去重+ 3. 对于同名实体,调用 LLM 进行智能合并++ :param data_list: 多个 EntityList 对象+ :return: 合并后的实体数据+ """+ if not data_list:+ logger.warning("Empty data list for merge")+ return self.data++ total_entities = 0+ duplicate_count = 0++ # 遍历新提取的数据,与已有的 entity_map 合并+ for entity_list_obj in data_list:+ for entity in entity_list_obj.entities:+ total_entities += 1+ entity_name = entity.name.strip()++ if not entity_name: # 跳过空名称+ logger.warning(f"Skipping entity with empty name: {entity}")+ continue++ if entity_name in self._entity_map:+ # 实体已存在 - 使用 LLM 执行合并+ duplicate_count += 1+ existing = self._entity_map[entity_name]+ merged = self._merge_entity_with_llm(existing, entity)+ if merged:+ self._entity_map[entity_name] = merged+ else:+ logger.warning(+ f"LLM merge failed for entity '{entity_name}', keeping existing"+ )+ else:+ # 新实体 - 深拷贝后添加+ self._entity_map[entity_name] = entity.model_copy(deep=True)++ # 标记需要同步+ self._entity_map_dirty = True+ self._index_dirty = True++ logger.info(+ f"Merged {len(self._entity_map)} unique entities "+ f"from {total_entities} total (removed {duplicate_count} duplicates)"+ )++ return self.data++ def _merge_entity_with_llm(+ self, entity_a: BaseEntity, entity_b: BaseEntity+ ) -> BaseEntity:+ """+ 使用 LLM 智能合并两个同名实体。++ :param entity_a: 第一个实体+ :param entity_b: 第二个实体+ :return: 合并后的实体(失败返回第一个实体)+ """+ try:+ # 准备输入+ entity_a_json = entity_a.model_dump_json(indent=2)+ entity_b_json = entity_b.model_dump_json(indent=2)++ # 调用 LLM 合并+ merged_entity = self._merge_chain.invoke(+ {+ "entity_a": entity_a_json,+ "entity_b": entity_b_json,+ }+ )++ logger.debug(f"Successfully merged entity: {merged_entity.name}")+ return merged_entity++ except Exception as e:+ logger.error(f"Error merging entity '{entity_a.name}' with LLM: {e}")+ return entity_a++ # ==================== 查询与索引 ====================++ def build_index(self):+ """+ 为每个实体构建独立的向量索引文档。++ 每个实体 → 一个 Document:+ - page_content: "{name}: {description}"+ - metadata: {"entity": entity.model_dump()}+ """+ if len(self._entity_map) == 0:+ logger.warning("No entities to index")+ return++ if not self._index_dirty and self._index is not None:+ return # 索引已是最新++ documents = []+ for entity in self._entity_map.values():+ # 构建文档内容(用于向量化)+ content = self._build_entity_content(entity)++ # 保存完整实体数据到 metadata+ documents.append(+ Document(+ page_content=content,+ metadata={"entity": entity.model_dump(), "name": entity.name},+ )+ )++ try:+ from langchain_community.vectorstores import FAISS++ self._index = FAISS.from_documents(documents, self.embedder)+ self._index_dirty = False+ logger.info(f"Built FAISS index with {len(documents)} entities")+ except ImportError:+ logger.error("FAISS not available. Install: pip install faiss-cpu")+ raise++ def _build_entity_content(self, entity: BaseEntity) -> str:+ """+ 构建实体的文本表示(用于向量化)。++ 策略:+ - 基础:"{name}: {description}"+ - 可选:添加其他字段+ """+ content_parts = [entity.name]++ if entity.description:+ content_parts.append(entity.description)++ # 添加其他字段(如果用户扩展了)+ for field_name in entity.model_fields:+ if field_name not in ("name", "description"):+ value = getattr(entity, field_name, None)+ if value is not None:+ content_parts.append(f"{field_name}: {value}")++ return "\n".join(content_parts)++ def search(+ self, query: str, top_k: int = 3, return_raw: bool = False+ ) -> List[BaseEntity]:+ """+ 语义搜索实体。++ :param query: 查询字符串+ :param top_k: 返回结果数+ :param return_raw: 是否返回原始实体对象(默认返回字典)+ :return: 实体列表+ """+ if len(self.entities) == 0:+ logger.warning("No entities to search")+ return []++ # 确保索引已构建+ if self._index is None or self._index_dirty:+ self.build_index()++ # 向量搜索+ docs = self._index.similarity_search(query, k=top_k)++ # 恢复实体对象+ results = []+ for doc in docs:+ try:+ entity_data = doc.metadata["entity"]+ if return_raw:+ # 返回 Pydantic 对象+ entity = self._entity_schema.model_validate(entity_data)+ results.append(entity)+ else:+ # 返回字典+ results.append(entity_data)+ except Exception as e:+ logger.warning(f"Failed to restore entity: {e}")++ logger.info(f"Found {len(results)} entities for query: {query[:50]}...")+ return results++ def size(self) -> int:+ """返回实体数量(直接从 entity_map 读取)"""+ return len(self._entity_map)++ # ==================== 便捷属性 ====================++ @property+ def entities(self) -> List[BaseEntity]:+ """快速访问实体列表(触发懒更新)"""+ return self.data.entities++ @property+ def entity_schema(self) -> Type[BaseEntity]:+ """返回用户定义的实体 Schema"""+ return self._entity_schema++ @property+ def entity_names(self) -> List[str]:+ """返回所有实体名称(直接从 entity_map 读取)"""+ return list(self._entity_map.keys())++ # ==================== 可选增强方法 ====================++ def get_entity_by_name(self, name: str) -> BaseEntity:+ """按名称获取实体(O(1) 字典查找)"""+ return self._entity_map.get(name)++ def remove_entity(self, name: str) -> bool:+ """删除指定实体(直接从 entity_map 删除)"""+ if name in self._entity_map:+ del self._entity_map[name]+ self._entity_map_dirty = True+ self._index_dirty = True+ logger.info(f"Removed entity: {name}")+ return True+ return False++ # ==================== 序列化 ====================++ def load(self, folder_path: str, **kwargs):+ """加载后重建 entity_map"""+ super().load(folder_path, **kwargs)++ # 从 _data 重建 entity_map+ self._entity_map = {+ entity.name: entity.model_copy(deep=True)+ for entity in self._data.entities+ }+ self._entity_map_dirty = False++ logger.info(f"Loaded {len(self._entity_map)} entities")diff --git a/src/knowledge/generic.py b/src/knowledge/generic.pyindex 88c3f67..2b939ca 100644--- a/src/knowledge/generic.py+++ b/src/knowledge/generic.py@@ -59,7 +59,7 @@ class GenericKnowledge(BaseKnowledge[T]):data_schema,llm_client,embedder,- prompt=prompt,+ prompt=prompt or self._default_prompt(),chunk_size=chunk_size,chunk_overlap=chunk_overlap,max_workers=max_workers,@@ -67,25 +67,30 @@ class GenericKnowledge(BaseKnowledge[T]):**kwargs,)+ @staticmethod+ def _default_prompt() -> str:+ """默认提取提示词"""+ return (+ "You are an expert knowledge extraction assistant. "+ "Your task is to carefully analyze the following text and extract structured information "+ "according to the specified schema. Be precise, comprehensive, and faithful to the source text. "+ "Extract all relevant details without adding information not present in the text.\n\n"+ "### Source Text:\n"+ )+# ==================== 数据管理 ====================def clear(self):"""清空所有知识"""- try:- self._data = self.data_schema()- except Exception:- self._data = self.data_schema.model_construct()-self.metadata["updated_at"] = datetime.now()-+ self._data = self.data_schema()self._index = Noneself._index_dirty = True-logger.info("Cleared all knowledge")# ==================== 提取与聚合 ====================- def extract(self, text: str) -> Dict[str, Any]:+ def extract(self, text: str) -> T:"""使用 LangChain 原生批处理提取知识"""start_time = datetime.now()@@ -96,7 +101,7 @@ class GenericKnowledge(BaseKnowledge[T]):logger.info(f"Processing single text (length: {len(text)})...")self._data = self.llm_chain.invoke({"chunk_text": text})self._index_dirty = True- extracted_data_list = [self._data]+ extracted_data = self._dataelse:# 长文本:分块提取chunks = self.text_splitter.split_text(text)@@ -113,37 +118,34 @@ class GenericKnowledge(BaseKnowledge[T]):logger.info(f"Extracted {len(extracted_data_list)} chunks")logger.info("Merging extracted knowledge...")- self.merge(extracted_data_list)+ extracted_data = self.merge(extracted_data_list)# 更新元数据self.metadata["updated_at"] = datetime.now()- duration = (datetime.now() - start_time).total_seconds()- return {"success": True, "duration_seconds": duration}+ logger.info("Knowledge extraction completed")+ logger.info(f"Duration: {(datetime.now() - start_time).total_seconds():.2f} seconds")+ return extracted_data- def merge(self, data_list: List[T]) -> Dict[str, Any]:+ def merge(self, data_list: List[T]) -> T:"""增量合并:第一次提取的结果为准。使用 model_copy(update=...) 实现第一次优先合并。:param data_list: 从各 chunk 提取的结果列表- :return: 合并状态+ :return: 合并后的知识数据"""- try:- self._data = data_list[0].model_copy()+ self._data = data_list[0].model_copy()- for item in data_list[1:]:- self._data = item.model_copy(- update=self._data.model_dump(exclude_none=True)- )+ for item in data_list[1:]:+ self._data = item.model_copy(+ update=self._data.model_dump(exclude_none=True)+ )- self._index_dirty = True- return True- except Exception as e:- logger.error(f"Error during merge: {e}")- return False+ self._index_dirty = True+ return self.data# ==================== 查询与索引 ====================diff --git a/src/knowledge/relation.py b/src/knowledge/relation.pynew file mode 100644index 0000000..e69de29