Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
重命名 ItemKnowledge 为 UnitKnowledge,并更新相关文档和代码以支持新的合并模式
yifanfeng971fe5defb71a9
diff --git a/README.md b/README.mdindex 922e872..9f26b8f 100644--- a/README.md+++ b/README.md@@ -37,7 +37,7 @@ Every knowledge container in Hyper-Extract follows a unified lifecycle:- **Field-Level Search**: Search across all schema fields with metadata preservation### 📦 Multiple Knowledge Types (Roadmap)-- ✅ **ItemKnowledge**: Universal schema-agnostic knowledge extraction+- ✅ **UnitKnowledge**: Universal schema-agnostic knowledge extraction- 🚧 **EntityKnowledge**: Entity-centric extraction with attribute management- 🚧 **RelationKnowledge**: Relationship extraction with entity linking- 🚧 **GraphKnowledge**: Knowledge graph construction (nodes + edges)@@ -90,14 +90,14 @@ class ArticleKnowledge(BaseModel):```pythonfrom langchain_openai import ChatOpenAI, OpenAIEmbeddings-from src.knowledge.generic import ItemKnowledge+from src.knowledge.generic import UnitKnowledge# Initialize LLM and embedderllm = ChatOpenAI(model="gpt-4o-mini", temperature=0)embedder = OpenAIEmbeddings(model="text-embedding-3-small")# Create knowledge container-knowledge = ItemKnowledge(+knowledge = UnitKnowledge(data_schema=ArticleKnowledge,llm_client=llm,embedder=embedder,@@ -167,7 +167,7 @@ knowledge.load("./output/my_knowledge")Check out the `examples/` directory for complete working examples:-- [`generic_knowledge_demo.py`](examples/generic_knowledge_demo.py) - Full ItemKnowledge walkthrough+- [`generic_knowledge_demo.py`](examples/generic_knowledge_demo.py) - Full UnitKnowledge walkthrough- More examples coming soon!---@@ -176,7 +176,7 @@ Check out the `examples/` directory for complete working examples:### Phase 1: Foundation (Current) ✅- [x] Abstract `BaseKnowledge` class-- [x] `ItemKnowledge` implementation+- [x] `UnitKnowledge` implementation- [x] LangChain integration with structured output- [x] FAISS vector search- [x] Serialization (dump/load)diff --git a/examples/generic_knowledge_demo.py b/examples/generic_knowledge_demo.pyindex 6189d1b..6b6da38 100644--- a/examples/generic_knowledge_demo.py+++ b/examples/generic_knowledge_demo.py@@ -1,7 +1,7 @@"""-ItemKnowledge 使用示例+UnitKnowledge 使用示例-演示如何使用 ItemKnowledge 类进行知识提取、搜索、保存和加载。+演示如何使用 UnitKnowledge 类进行知识提取、搜索、保存和加载。"""import sys@@ -14,7 +14,7 @@ sys.path.insert(0, str(project_root))from pydantic import BaseModel, Fieldfrom typing import Listfrom langchain_openai import ChatOpenAI, OpenAIEmbeddings-from src.knowledge.generic import ItemKnowledge+from src.knowledge.generic import UnitKnowledge# 1. 定义知识 Schema@@ -30,7 +30,7 @@ class ArticleKnowledge(BaseModel):def main():print("=" * 60)- print("ItemKnowledge 使用示例")+ print("UnitKnowledge 使用示例")print("=" * 60)# 2. 初始化 LLM 和 Embedder@@ -38,9 +38,9 @@ def main():llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)embedder = OpenAIEmbeddings(model="text-embedding-3-small")- # 3. 创建 ItemKnowledge 实例- print("[2] 创建 ItemKnowledge 实例...")- knowledge = ItemKnowledge(+ # 3. 创建 UnitKnowledge 实例+ print("[2] 创建 UnitKnowledge 实例...")+ knowledge = UnitKnowledge(data_schema=ArticleKnowledge,llm_client=llm,embedder=embedder,@@ -104,7 +104,7 @@ def main():# 9. 加载知识print("\n[8] 从文件夹加载知识...")- new_knowledge = ItemKnowledge(+ new_knowledge = UnitKnowledge(data_schema=ArticleKnowledge,llm_client=llm,embedder=embedder,diff --git a/src/knowledge/base.py b/src/knowledge/base.pyindex f7d6b51..c9e307d 100644--- a/src/knowledge/base.py+++ b/src/knowledge/base.py@@ -122,21 +122,22 @@ class BaseKnowledge(ABC, Generic[T]):# ==================== 提取与聚合 ====================@abstractmethod- def extract(self, text: str, *, append_mode: bool = False) -> T:+ def extract(self, text: str, *, merge_mode: bool = False) -> T:"""主提取方法 - 自动处理长文本分块、提取、聚合。流程:- 1. 根据 append_mode 决定是否清空现有数据+ 1. 根据 merge_mode 决定是否清空现有数据2. 判断文本长度,决定是否分块3. 对每个块进行提取- 4. 调用 merge() 将提取结果聚合到 self._data- 5. 返回提取到的结果+ 4. 调用 merge() 合并数据(merge 不修改内部状态)+ 5. extract 负责将 merge 的返回值赋值给 self._data 并更新 self._index_dirty+ 6. 返回提取到的结果:param text: 输入文本(可以是短文本或长文本)- :param append_mode: 是否累积模式(默认 False)- - False(默认): 替换模式 - 先清空现有知识,再提取新数据- - True: 累积模式 - 保留现有知识,合并新数据+ :param merge_mode: 合并模式(默认 False)+ - False(默认): 替换模式 - 仅合并新提取的数据+ - True: 累积模式 - 将现有知识与新数据合并:return: 提取到的知识数据"""pass@@ -144,13 +145,15 @@ class BaseKnowledge(ABC, Generic[T]):@abstractmethoddef merge(self, data_list: List[T]) -> T:"""- 将多个提取结果合并到内部状态 self._data。+ 纯数据合并方法 - 不修改内部状态。- 职责:去重、合并、冲突解决。- 子类必须实现此方法.+ 职责:+ - 实现具体的合并算法(去重、冲突解决等)+ - 返回合并后的新数据对象+ - 不修改任何实例属性- :param data_list: 从各个块提取的知识列表- :return: 合并后的知识数据+ :param data_list: 需要合并的数据列表+ :return: 合并后的知识数据(新对象)"""passdiff --git a/src/knowledge/entity.py b/src/knowledge/entity.pyindex f109bd9..3ac86c8 100644--- a/src/knowledge/entity.py+++ b/src/knowledge/entity.py@@ -130,33 +130,29 @@ Please merge these two entities intelligently and return the result."""# ==================== 提取与聚合 ====================- def extract(self, text: str, *, append_mode: bool = False) -> BaseModel:+ def extract(self, text: str, *, merge_mode: bool = False) -> BaseModel:"""提取实体知识(支持替换/累积两种模式)。模式说明:- - append_mode=False(默认): 替换模式- 先清空现有实体,然后提取新实体+ - merge_mode=False(默认): 替换模式+ 仅使用新提取的实体- - append_mode=True: 累积模式+ - merge_mode=True: 累积模式保留现有实体,提取新实体并智能合并(LLM 去重):param text: 输入文本- :param append_mode: 是否累积模式(默认 False)+ :param merge_mode: 合并模式(默认 False):return: 提取后的实体列表容器"""- # 替换模式:先清空- if not append_mode:- self.clear()-- # 调用父类提取方法- result = super().extract(text)+ # 调用父类提取方法,传递 merge_mode+ result = super().extract(text, merge_mode=merge_mode)# 提取完成后同步到 entity_mapself._sync_items_to_map()logger.info(- f"Entity extraction completed ({'append' if append_mode else 'replace'} mode)"+ f"Entity extraction completed ({'累积' if merge_mode else '替换'} mode)")logger.info(f"Total entities: {len(self._entity_map)}")@@ -168,20 +164,24 @@ Please merge these two entities intelligently and return the result."""策略:1. 先调用父类 merge 获得所有提取的 items- 2. 使用 name 作为唯一标识符去重- 3. 对于同名实体,调用 LLM 进行智能合并- 4. 更新 self.items 为去重后的列表+ 2. 将结果赋值给 self._data+ 3. 同步到 entity_map 进行 LLM 去重+ 4. 返回去重后的数据:param data_list: 多个容器对象:return: 合并后的实体数据"""- # 调用父类 merge,获得所有原始 items- super().merge(data_list)+ # 调用父类 merge,获得合并后的结果+ merged_result = super().merge(data_list)++ # 临时赋值给 self._data 以便 _sync_items_to_map 能访问 self.items+ self._data = merged_result# 同步到 entity_map 进行去重self._sync_items_to_map()- return self.data+ # 返回最终的 self._data(已经被 _sync_items_to_map 更新)+ return self._datadef _sync_items_to_map(self):"""diff --git a/src/knowledge/generic.py b/src/knowledge/generic.pyindex 282b443..879206e 100644--- a/src/knowledge/generic.py+++ b/src/knowledge/generic.py@@ -2,8 +2,8 @@基础知识模式实现。包含两种核心模式:-- ItemKnowledge: 单对象模式(适用于摘要、元数据等)-- ListKnowledge: 列表模式(适用于实体、事件等列表提取)+- UnitKnowledge: 单元知识模式(适用于摘要、元数据等)+- ListKnowledge: 列表知识模式(适用于实体、事件等列表提取)"""import json@@ -25,10 +25,14 @@ except ImportError:logger = logging.getLogger(__name__)+# ==================== 两种基础知识模式 ====================++# UnitKnowledge: 单元知识模式+-class ItemKnowledge(BaseKnowledge[T]):+class UnitKnowledge(BaseKnowledge[T]):"""- 单例知识模式 (Single Item Pattern) - 适用于任何 Pydantic Schema。+ 单元知识模式 (Unit Knowledge Pattern) - 适用于任何 Pydantic Schema。特点:- 针对整篇文档提取 **唯一** 的一个结构化对象@@ -95,17 +99,33 @@ class ItemKnowledge(BaseKnowledge[T]):# ==================== 提取与聚合 ====================- def extract(self, text: str) -> T:- """使用 LangChain 原生批处理提取知识"""+ def extract(self, text: str, *, merge_mode: bool = False) -> T:+ """+ 使用 LangChain 原生批处理提取知识++ :param text: 输入文本+ :param merge_mode: 合并模式(默认 False)+ - False: 替换模式 - 清空后仅使用新提取的数据+ - True: 累积模式 - 将现有数据与新数据合并+ :return: 提取到的知识数据+ """start_time = datetime.now()+ # 替换模式:先清空+ if not merge_mode:+ self.clear()+# 判断是否需要分块if len(text) <= self.chunk_size:+ # 短文本:直接提取if self.show_progress:logger.info(f"Processing single text (length: {len(text)})...")- self._data = self.llm_chain_extract.invoke({"chunk_text": text})- self._index_dirty = True++ extracted_data = self.llm_chain_extract.invoke({"chunk_text": text})+ extracted_data_list = [extracted_data]+else:+ # 长文本:分块提取chunks = self.text_splitter.split_text(text)logger.info(f"Split text into {len(chunks)} chunks")@@ -116,13 +136,18 @@ class ItemKnowledge(BaseKnowledge[T]):extracted_data_list = self.llm_chain_extract.batch(inputs, config={"max_concurrency": self.max_workers})- if self.show_progress:- logger.info(f"Extracted {len(extracted_data_list)} chunks")- logger.info("Merging extracted knowledge...")- self.merge(extracted_data_list)+ if self.show_progress:+ logger.info(f"Extracted {len(extracted_data_list)} chunks")- # 更新元数据+ logger.info("Merging extracted knowledge...")+ if merge_mode:+ extracted_data_list.insert(0, self.data)++ self._data = self.merge(extracted_data_list)++ # 统一修改状态+ self._index_dirty = Trueself.metadata["updated_at"] = datetime.now()logger.info("Knowledge extraction completed")@@ -133,23 +158,20 @@ class ItemKnowledge(BaseKnowledge[T]):def merge(self, data_list: List[T]) -> T:"""- 增量合并:第一次提取的结果为准。+ 纯数据合并方法 - 字段级更新策略。+ 合并策略:第一次提取的结果为准,后续只能补充缺失字段。使用 model_copy(update=...) 实现第一次优先合并。- :param data_list: 从各 chunk 提取的结果列表- :return: 合并后的知识数据+ :param data_list: 需要合并的数据列表+ :return: 合并后的知识数据(新对象)"""-- self._data = data_list[0].model_copy()+ result = data_list[0].model_copy()for item in data_list[1:]:- self._data = item.model_copy(- update=self._data.model_dump(exclude_none=True)- )+ result = item.model_copy(update=result.model_dump(exclude_none=True))- self._index_dirty = True- return self.data+ return result# ==================== 查询与索引 ====================@@ -317,10 +339,8 @@ class ItemKnowledge(BaseKnowledge[T]):logger.info(f"Loaded knowledge successfully with {self.size()} fields")-# ==================== 两种基础知识模式 ====================+# ==================== 列表知识模式 ===================-# ItemKnowledge: 单对象模式(已在上方定义)-# ListKnowledge: 列表模式(如下)Item = TypeVar("Item", bound=BaseModel)@@ -339,8 +359,8 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):- 合并策略:追加 (Append) + 基础去重(可由子类扩展)- 索引策略:对列表中的每个 Item 独立建立索引- 与 ItemKnowledge 的区别:- - ItemKnowledge: 提取单个结构化对象(如摘要、元数据)+ 与 UnitKnowledge 的区别:+ - UnitKnowledge: 提取单个结构化对象(如摘要、元数据)- ListKnowledge: 提取多个独立对象的列表(如实体列表、事件列表)"""@@ -370,10 +390,10 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):self.item_schema = item_schema# 使用显式定义的 ItemListSchema- self.container_schema = ItemListSchema[item_schema]+ self.item_list_schema = ItemListSchema[item_schema]super().__init__(- data_schema=self.container_schema,+ data_schema=self.item_list_schema,llm_client=llm_client,embedder=embedder,prompt=prompt or self._default_prompt(),@@ -403,23 +423,36 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):def clear(self):"""清空所有列表项"""self.metadata["updated_at"] = datetime.now()- self._data = self.container_schema(items=[])+ self._data = self.item_list_schema(items=[])self._index = Noneself._index_dirty = Truelogger.info("Cleared list knowledge")- def extract(self, text: str) -> BaseModel:- """使用 LangChain 原生批处理提取列表"""+ def extract(self, text: str, *, merge_mode: bool = False) -> BaseModel:+ """+ 使用 LangChain 原生批处理提取列表++ :param text: 输入文本+ :param merge_mode: 合并模式(默认 False)+ - False: 替换模式 - 清空后仅使用新提取的数据+ - True: 累积模式 - 将现有数据与新数据合并+ :return: 提取到的 items 列表+ """start_time = datetime.now()+ # 替换模式:先清空+ if not merge_mode:+ self.clear()+# 判断是否需要分块if len(text) <= self.chunk_size:# 短文本:一次性提取if self.show_progress:logger.info(f"Processing single text (length: {len(text)})...")- result = self.llm_chain_extract.invoke({"chunk_text": text})- self._data = result- self._index_dirty = True++ extracted_data = self.llm_chain_extract.invoke({"chunk_text": text})+ extracted_data_list = [extracted_data]+else:# 长文本:分块提取chunks = self.text_splitter.split_text(text)@@ -432,39 +465,45 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):extracted_data_list = self.llm_chain_extract.batch(inputs, config={"max_concurrency": self.max_workers})- if self.show_progress:- logger.info(f"Extracted {len(extracted_data_list)} chunks")- logger.info("Merging extracted knowledge...")- self.merge(extracted_data_list)+ if self.show_progress:+ logger.info(f"Extracted {len(extracted_data_list)} chunks")- # 更新元数据+ logger.info("Merging extracted knowledge...")+ if merge_mode:+ extracted_data_list.insert(0, self.data)++ self._data = self.merge(extracted_data_list)++ # 统一修改状态+ self._index_dirty = Trueself.metadata["updated_at"] = datetime.now()logger.info("Knowledge extraction completed")logger.info(f"Duration: {(datetime.now() - start_time).total_seconds():.2f} seconds")- return self.data+ return self.items- def merge(self, data_list: List[BaseModel]) -> BaseModel:+ def merge(self, data_list: List[ItemListSchema]) -> ItemListSchema:"""- 合并策略:列表追加 (Append)。- 收集所有 Chunk 提取出的 items,合并到一个大列表中。+ 纯数据合并方法 - 列表追加策略。++ 合并策略:收集所有容器对象中的 items,合并到一个大列表中。子类可以重写此方法实现更复杂的去重逻辑(如 EntityKnowledge)。- :param data_list: 从各 chunk 提取的结果列表(每个都是容器对象)- :return: 合并后的知识数据+ :param data_list: 需要合并的容器对象列表+ :return: 合并后的知识数据(新的 ItemListSchema 对象)"""all_items = []- for container in data_list:- if hasattr(container, "items") and container.items:- all_items.extend(container.items)- # 简单追加(子类可以重写实现去重)- self._data.items = all_items- self._index_dirty = True- return self.data+ # 收集所有 items+ for data in data_list:+ copied_data = data.model_copy(deep=True)+ all_items.extend(copied_data.items)++ # 返回新的对象+ return self.item_list_schema(items=all_items)def build_index(self):"""为列表中的每个 Item 构建独立索引"""@@ -534,7 +573,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):def dump(self, folder_path: str) -> Any:"""- 导出知识到指定文件夹(复用 ItemKnowledge 的实现)。+ 导出知识到指定文件夹(复用 UnitKnowledge 的实现)。容器本质也是 BaseModel,可以直接使用相同的序列化逻辑。"""from pathlib import Path@@ -544,7 +583,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):# 1. 保存结构化数据data = {- "schema_name": self.container_schema.__name__,+ "schema_name": self.item_list_schema.__name__,"item_schema_name": self.item_schema.__name__,"item_schema": self.item_schema.model_json_schema(),"data": self.data.model_dump(),@@ -587,7 +626,7 @@ class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):data = json.load(f)logger.info(f"Loaded data from {data_file}")- self._data = self.container_schema.model_validate(data.get("data", {}))+ self._data = self.item_list_schema.model_validate(data.get("data", {}))# 更新元数据if "metadata" in data: