Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
feat: Introduce knowledge graph and hypergraph modules
yifanfeng97783da831a808
diff --git a/.gitignore b/.gitignoreindex c8fd3ff..5b11d71 100644--- a/.gitignore+++ b/.gitignore@@ -1,4 +1,5 @@tmp/+logs/# Byte-compiled / optimized / DLL files__pycache__/diff --git a/README.md b/README.mdindex 9f26b8f..c73e5d2 100644--- a/README.md+++ b/README.md@@ -90,7 +90,7 @@ class ArticleKnowledge(BaseModel):```pythonfrom langchain_openai import ChatOpenAI, OpenAIEmbeddings-from src.knowledge.generic import UnitKnowledge+from hyperextract.knowledge.generic import UnitKnowledge# Initialize LLM and embedderllm = ChatOpenAI(model="gpt-4o-mini", temperature=0)diff --git a/README_old.md b/README_old.mddeleted file mode 100644index b8dcb26..0000000--- a/README_old.md+++ /dev/null@@ -1,111 +0,0 @@--# Hyper-Extract 🧠--> **An Intelligent, LLM-Driven Knowledge Container Framework.**->-> 一个基于 LLM 的智能知识提取与演化框架。不仅仅是存储数据,它能“阅读”、“思考”并“进化”。--## 📖 简介 (Introduction)--**Hyper-Extract** 是一个重新定义的非结构化数据处理框架。与传统的 ETL 工具不同,Hyper-Extract 的核心单元是 **Knowledge Container(知识容器)**。--每个容器(如 `SimpleGraph`, `HyperGraph`)都持有一个 **LLM 实例**。它们不仅仅是被动的数据结构,而是具有生命周期的智能体:--1. **Extract (摄入)**: 主动利用 LLM 从文本中提取信息,并执行去重、对齐、合并。-2. **Evolve (演化)**: 对已有知识进行内省(Introspection)。包括推理新关系、剪枝错误节点、图卷积传播等。-3. **Dump (产出)**: 将清洗后的高质量知识导出为标准格式(JSON/NetworkX/Neo4j)。--## ✨ 核心特性 (Features)--* **LLM Native**: 容器初始化即绑定 LangChain 模型实例,所有操作皆由大模型驱动。-* **统一生命周期**: 所有知识类型遵循 `Extract` -> `Evolve` -> `Dump` 的标准范式。-* **多模态结构支持**:- * **Graph**: 标准实体关系图谱 (Nodes + Edges)。- * **Hypergraph**: 支持超边(Hyperedge)连接多个节点的复杂语义结构 (HyperRAG 实现)。-* **类型安全**: 全面基于 Pydantic Schema,确保结构化输出的稳定性。--## 🛠️ 安装 (Installation)--本项目使用 `uv` 进行包管理(也支持标准 pip)。--```bash-# 克隆项目-git clone https://github.com/your-username/hyper-extract.git-cd hyper-extract--# 使用 uv 安装依赖 (推荐)-uv sync--# 或者使用 pip-pip install -r requirements.txt-```--## 🚀 快速开始 (Quick Start)--### 1. 定义与初始化--```python-import asyncio-from langchain_openai import ChatOpenAI-from src.knowledge.graph.simple_graph import SimpleGraph--async def main():- # 1. 准备大模型实例- llm = ChatOpenAI(model="gpt-4o", temperature=0)-- # 2. 初始化智能容器- # 容器就像一个空的大脑,准备好接收信息- kg = SimpleGraph(llm=llm)-- # 3. Extract: 摄入知识- text = "Elon Musk co-founded OpenAI in 2015. He later left the board in 2018."- print(f"正在提取: {text}...")- await kg.extract(text)-- # 4. Evolve: 知识演化 (可选)- # 例如:推断隐含关系或清理孤立节点- await kg.evolve()-- # 5. Dump: 导出结果- data = kg.dump(format="json")- print("最终知识图谱:", data)--if __name__ == "__main__":- asyncio.run(main())-```--## 📂 项目结构 (Project Structure)--```text-hyper-extract/-├── src/-│ ├── core/ # 核心抽象定义-│ ├── driver/ # LLM 驱动与适配器-│ ├── schema/ # Pydantic 数据结构定义 (GraphSchema, etc.)-│ └── knowledge/ # 知识容器实现-│ ├── base.py # [核心] BaseKnowledge 基类-│ ├── graph/ # 普通图谱实现-│ │ ├── simple_graph.py # 基础图提取-│ │ └── kg_gen_graph.py # 复杂生成式图谱-│ └── hypergraph/ # 超图实现-│ ├── simple_hypergraph.py-│ └── hyperrag_hypergraph.py-├── tests/ # 测试用例-├── main.py # 入口示例-├── pyproject.toml # 项目配置-└── uv.lock # 依赖锁定-```--## 🧩 扩展指南 (Development)--如果你想添加一种新的知识结构(例如“思维导图”):--1. 继承 `src.knowledge.base.BaseKnowledge`。-2. 定义你的数据 Schema (Pydantic)。-3. 实现 **`extract`**: 定义如何用 Prompt 将文本转化为你的 Schema。-4. 实现 **`evolve`** (可选): 定义如何优化你的数据结构。-5. 实现 **`dump`**: 定义导出格式。--## 📄 License--MIT License\ No newline at end of filediff --git a/examples/generic_knowledge_demo.py b/examples/generic_knowledge_demo.pyindex 7da8bb0..1a6bb81 100644--- a/examples/generic_knowledge_demo.py+++ b/examples/generic_knowledge_demo.py@@ -14,7 +14,7 @@ sys.path.insert(0, str(project_root))from pydantic import BaseModel, Fieldfrom typing import Listfrom langchain_openai import ChatOpenAI, OpenAIEmbeddings-from src.knowledge.generic import UnitKnowledge+from hyperextract.knowledge.generic import UnitKnowledge# 1. 定义知识 Schemadiff --git a/examples/list_knowledge_multi_chunk_demo.py b/examples/list_knowledge_multi_chunk_demo.pyindex 2880108..31b5de8 100644--- a/examples/list_knowledge_multi_chunk_demo.py+++ b/examples/list_knowledge_multi_chunk_demo.py@@ -14,7 +14,7 @@ sys.path.insert(0, str(project_root))from pydantic import BaseModel, Fieldfrom typing import Optionalfrom langchain_openai import ChatOpenAI, OpenAIEmbeddings-from src.knowledge.generic import ListKnowledge+from hyperextract.knowledge.generic import ListKnowledge# 定义单个新闻事件的 Schema@@ -151,10 +151,10 @@ def main():show_progress=True,)- # 4. 提取知识 (替换模式)- print("\n[4] 提取新闻事件列表 (替换模式)...")+ # 4. 提取知识 (默认存储模式)+ print("\n[4] 提取新闻事件列表 (默认存储模式)...")print("-" * 80)- events = knowledge.extract(news_text, merge_mode=False)+ events = knowledge.extract(news_text)print("-" * 80)# 5. 显示提取结果@@ -194,8 +194,8 @@ def main():print(f" [{i}] {event.title} ({event.date})")print(f" 地点: {event.location}")- # 9. 测试增量提取(合并模式)- print("\n[9] 测试增量提取(合并模式)...")+ # 9. 测试增量提取(默认合并模式)+ print("\n[9] 测试增量提取(默认合并模式)...")additional_news = """### 事件11: 华为发布6G测试网络2024年6月15日,华为在深圳总部宣布建成全球首个6G技术测试网络。@@ -205,7 +205,7 @@ def main():print(" 添加补充新闻...")before_count = len(knowledge)- knowledge.extract(additional_news, merge_mode=True)+ knowledge.extract(additional_news) # 默认 store=True,自动合并after_count = len(knowledge)print(f" 合并前: {before_count} 个事件")print(f" 合并后: {after_count} 个事件")@@ -217,6 +217,7 @@ def main():for event in knowledge.items[-3:]:print(f" - {event.title} ({event.date})")+ knowledge.build_index()# 11. 保存知识print("\n[10] 保存知识到文件...")save_path = Path(__file__).parent.parent / "tmp" / "list_knowledge_multi_chunk"diff --git a/examples/unit_knowledge_multi_chunk_demo.py b/examples/unit_knowledge_multi_chunk_demo.pyindex 0ccba2b..0435f74 100644--- a/examples/unit_knowledge_multi_chunk_demo.py+++ b/examples/unit_knowledge_multi_chunk_demo.py@@ -14,7 +14,7 @@ sys.path.insert(0, str(project_root))from pydantic import BaseModel, Fieldfrom typing import List, Optionalfrom langchain_openai import ChatOpenAI, OpenAIEmbeddings-from src.knowledge.generic import UnitKnowledge+from hyperextract.knowledge.generic import UnitKnowledge# 定义文档知识 Schema@@ -161,10 +161,10 @@ def main():show_progress=True,)- # 4. 提取知识 (替换模式)- print("\n[4] 提取知识 (替换模式)...")+ # 4. 提取知识 (默认存储模式)+ print("\n[4] 提取知识 (默认存储模式)...")print("-" * 80)- extracted_data = knowledge.extract(document, merge_mode=False)+ extracted_data = knowledge.extract(document)print("-" * 80)# 5. 显示提取结果@@ -203,15 +203,15 @@ def main():field_value = result[field_name]print(f" [{i}] {field_name}: {str(field_value)[:80]}...")- # 9. 测试增量提取(合并模式)- print("\n[9] 测试增量提取(合并模式)...")+ # 9. 测试增量提取(默认合并模式)+ print("\n[9] 测试增量提取(默认合并模式)...")additional_text = """补充信息:本研究获得了国家自然科学基金支持,项目编号:NSFC-2023-12345。研究团队来自清华大学医学院和北京协和医院。"""print(" 添加补充文本...")- knowledge.extract(additional_text, merge_mode=True)+ knowledge.extract(additional_text) # 默认 store=True,自动合并print(f" 更新后的作者信息: {knowledge.data.author}")# 10. 保存知识diff --git a/src/__init__.py b/hyperextract/__init__.pysimilarity index 100%rename from src/__init__.pyrename to hyperextract/__init__.pydiff --git a/src/config.py b/hyperextract/config.pysimilarity index 100%rename from src/config.pyrename to hyperextract/config.pydiff --git a/hyperextract/knowledge/__init__.py b/hyperextract/knowledge/__init__.pynew file mode 100644index 0000000..518926d--- /dev/null+++ b/hyperextract/knowledge/__init__.py@@ -0,0 +1,45 @@+"""Knowledge extraction framework.++This package provides a hierarchical knowledge extraction framework:+ - generic: Base knowledge patterns (UnitKnowledge, ListKnowledge, SetKnowledge)+ - typical: Common knowledge types (EntityKnowledge, RelationKnowledge)+ - domain: Domain-specific knowledge patterns (reserved for future use)+ - graph: Graph-based knowledge structures+ - hypergraph: Hypergraph-based knowledge structures++Usage:+ All main classes can be imported directly from hyperextract.knowledge:++ >>> from hyperextract.knowledge import UnitKnowledge, ListKnowledge, EntityKnowledge+ >>> from hyperextract.knowledge import SimpleGraph+"""++from .base import BaseKnowledge++# Generic patterns+from .generic import UnitKnowledge, ListKnowledge, ItemListSchema++# Typical patterns+from .typical import EntityKnowledge++# Graph patterns+from .graph.base import SimpleGraph, Entity, Relationship, GraphSchema++__all__ = [+ # Base+ "BaseKnowledge",++ # Generic patterns+ "UnitKnowledge",+ "ListKnowledge",+ "ItemListSchema",++ # Typical patterns+ "EntityKnowledge",++ # Graph patterns+ "SimpleGraph",+ "Entity",+ "Relationship",+ "GraphSchema",+]diff --git a/src/knowledge/base.py b/hyperextract/knowledge/base.pysimilarity index 71%rename from src/knowledge/base.pyrename to hyperextract/knowledge/base.pyindex c6adf12..186299a 100644--- a/src/knowledge/base.py+++ b/hyperextract/knowledge/base.py@@ -129,25 +129,25 @@ class BaseKnowledge(ABC, Generic[T]):# ==================== Extraction & Aggregation ====================@abstractmethod- def extract(self, text: str, *, merge_mode: bool = False) -> T:+ def extract(self, text: str, *, store: bool = True) -> T:"""Extracts structured knowledge from text with automatic chunking and aggregation.This is the main entry point for knowledge extraction. It handles the complete pipelinefrom text preprocessing to knowledge aggregation.Extraction pipeline:- 1. Decide whether to clear existing data based on merge_mode- 2. Determine if text chunking is needed based on length- 3. Extract knowledge from each chunk using LLM- 4. Merge all extracted data using the merge() method- 5. Update internal state (_data) and invalidate index via clear_index()+ 1. Determine if text chunking is needed based on length+ 2. Extract knowledge from each chunk using LLM+ 3. Merge all extracted data using the merge() method+ 4. If store=True: merge with existing data and update internal state+ 5. If store=False: return extracted data without modifying internal state6. Return the aggregated knowledgeArgs:text: Input text to extract knowledge from (can be short or long).- merge_mode: Controls how new data is combined with existing knowledge.- - False (default): Replace mode - only use newly extracted data- - True: Accumulative mode - merge new data with existing knowledge+ store: Controls whether to store extracted knowledge internally.+ - True (default): Store mode - merge with existing knowledge and update internal state+ - False: Temporary mode - return extracted data without modifying internal stateReturns:The extracted and aggregated knowledge data.@@ -174,6 +174,80 @@ class BaseKnowledge(ABC, Generic[T]):"""pass+ def _create_new_instance(self) -> "BaseKnowledge[T]":+ """Creates a new instance with the same configuration as this one.++ Subclasses can override this method if they have special initialization requirements.++ Returns:+ A new knowledge instance with the same configuration.+ """+ return self.__class__(+ data_schema=self._data_schema,+ llm_client=self.llm_client,+ embedder=self.embedder,+ prompt=self.prompt,+ chunk_size=self.chunk_size,+ chunk_overlap=self.chunk_overlap,+ max_workers=self.max_workers,+ show_progress=self.show_progress,+ )++ def __add__(self, other: "BaseKnowledge[T]") -> "BaseKnowledge[T]":+ """Operator overload for '+' to merge two knowledge instances.++ Creates a new knowledge instance by merging the data from both instances.+ The new instance inherits configuration from the left operand (self).++ Usage:+ >>> kb1 = UnitKnowledge(PersonSchema, ...)+ >>> kb2 = UnitKnowledge(PersonSchema, ...)+ >>> kb3 = kb1 + kb2 # ✅ Same schema, creates merged instance+ >>>+ >>> kb4 = UnitKnowledge(CompanySchema, ...)+ >>> kb5 = kb1 + kb4 # ❌ TypeError: Different schemas++ Args:+ other: Another knowledge instance of the same type to merge with.++ Returns:+ A new knowledge instance containing merged data.++ Raises:+ TypeError: If other is not an instance of the same knowledge class or has different data schema.+ """+ # Check 1: Both must be instances of the same knowledge class+ if not isinstance(other, self.__class__):+ raise TypeError(+ f"Cannot add {type(other).__name__} to {type(self).__name__}. "+ f"Both operands must be instances of the same knowledge class."+ )++ # Check 2: Both must have the same data schema+ if self._data_schema != other._data_schema:+ raise TypeError(+ f"Cannot add knowledge instances with different data schemas. "+ f"Left schema: {self._data_schema.__name__}, "+ f"Right schema: {other._data_schema.__name__}. "+ f"Both operands must have the same data schema to be merged."+ )++ # Merge the data from both instances+ merged_data = self.merge([self._data, other._data])++ # Create a new instance with the same configuration+ new_instance = self._create_new_instance()++ # Set the merged data and update metadata+ new_instance._data = merged_data+ new_instance.metadata["created_at"] = min(+ self.metadata["created_at"], other.metadata["created_at"]+ )+ new_instance.metadata["updated_at"] = datetime.now()+ new_instance.clear_index() # Index needs to be rebuilt++ return new_instance+# ==================== Evolution ====================def evolve(self, **kwargs) -> T:diff --git a/hyperextract/knowledge/domain/__init__.py b/hyperextract/knowledge/domain/__init__.pynew file mode 100644index 0000000..b3e6ce6--- /dev/null+++ b/hyperextract/knowledge/domain/__init__.py@@ -0,0 +1,11 @@+"""Domain-specific knowledge patterns.++This module is reserved for domain-specific knowledge implementations such as:+ - DocumentKnowledge: Document-level knowledge (title, author, sections, etc.)+ - ConversationKnowledge: Multi-turn conversation knowledge+ - AcademicPaperKnowledge: Academic paper structure and metadata+ - CodeKnowledge: Source code analysis and documentation+ - etc.+"""++__all__ = []diff --git a/hyperextract/knowledge/generic/__init__.py b/hyperextract/knowledge/generic/__init__.pynew file mode 100644index 0000000..a3d3851--- /dev/null+++ b/hyperextract/knowledge/generic/__init__.py@@ -0,0 +1,10 @@+"""Generic knowledge patterns - fundamental data structure patterns."""++from .unit import UnitKnowledge+from .list import ListKnowledge, ItemListSchema++__all__ = [+ "UnitKnowledge",+ "ListKnowledge",+ "ItemListSchema",+]diff --git a/hyperextract/knowledge/generic/list.py b/hyperextract/knowledge/generic/list.pynew file mode 100644index 0000000..927adac--- /dev/null+++ b/hyperextract/knowledge/generic/list.py@@ -0,0 +1,413 @@+"""List Knowledge Pattern - extracts a collection of objects from text."""++import json+from typing import List, Any, Type, TypeVar, Generic, TYPE_CHECKING+from datetime import datetime+from pathlib import Path+from pydantic import BaseModel, Field, create_model+from langchain_core.language_models.chat_models import BaseChatModel+from langchain_core.embeddings import Embeddings+from langchain_core.documents import Document+from langchain_community.vectorstores import FAISS++from ..base import BaseKnowledge++try:+ from hyperextract.config import logger+except ImportError:+ import logging+ logger = logging.getLogger(__name__)+++Item = TypeVar("Item", bound=BaseModel)+++class ItemListSchema(BaseModel, Generic[Item]):+ """Generic schema container for list-based knowledge patterns."""++ items: List[Item] = Field(default_factory=list, description="Item list")+++class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):+ """List Knowledge Pattern - extracts a collection of objects from text.++ This pattern extracts multiple independent objects from a document, suitable for+ extracting entities, events, references, or any collection of structured items.++ Key characteristics:+ - Extraction target: A collection of structured objects+ - Merge strategy: Append with basic deduplication (extensible by subclasses)+ - Indexing strategy: Each item in the list is indexed independently++ Comparison with UnitKnowledge:+ - UnitKnowledge: Extracts a single structured object (e.g., summary, metadata)+ - ListKnowledge: Extracts multiple independent objects (e.g., entity list, event list)+ """++ if TYPE_CHECKING:+ # Use generic version during type checking to maintain complete type hints+ item_list_schema: Type[ItemListSchema[Item]]++ def __init__(+ self,+ item_schema: Type[Item],+ llm_client: BaseChatModel,+ embedder: Embeddings,+ *,+ prompt: str = "",+ chunk_size: int = 2000,+ chunk_overlap: int = 200,+ max_workers: int = 10,+ show_progress: bool = True,+ **kwargs,+ ):+ """Initialize ListKnowledge with item schema and configuration.++ Args:+ item_schema: Pydantic BaseModel subclass for individual list items.+ llm_client: Language model client for extraction.+ embedder: Embedding model for vector indexing.+ prompt: Custom extraction prompt (defaults to list-oriented prompt).+ chunk_size: Maximum characters per chunk for long texts.+ chunk_overlap: Overlapping characters between adjacent chunks.+ max_workers: Maximum concurrent extraction tasks.+ show_progress: Whether to log progress information.+ """+ self.item_schema = item_schema++ container_name = f"{item_schema.__name__}List"+ self.item_list_schema = create_model(+ container_name,+ items=(+ List[item_schema],+ Field(default_factory=list, description="Item list"),+ ),+ )++ super().__init__(+ data_schema=self.item_list_schema,+ llm_client=llm_client,+ embedder=embedder,+ prompt=prompt or self._default_prompt(),+ chunk_size=chunk_size,+ chunk_overlap=chunk_overlap,+ max_workers=max_workers,+ show_progress=show_progress,+ **kwargs,+ )++ @staticmethod+ def _default_prompt() -> str:+ """Returns the default extraction prompt for list-based extraction."""+ return (+ "You are an expert knowledge extraction assistant. "+ "Extract all relevant items from the text into a list. "+ "Be comprehensive and ensure no item is missed. "+ "Extract all items without adding information not present in the text.\n\n"+ "### Source Text:\n"+ )++ @property+ def items(self) -> List[Item]:+ """Returns the internal list of extracted items."""+ return getattr(self._data, "items", [])++ def _create_new_instance(self) -> "ListKnowledge[Item]":+ """Creates a new instance with the same configuration as this one.++ Overrides base class method to handle ListKnowledge's item_schema parameter.++ Returns:+ A new ListKnowledge instance with the same configuration.+ """+ return self.__class__(+ item_schema=self.item_schema,+ llm_client=self.llm_client,+ embedder=self.embedder,+ prompt=self.prompt,+ chunk_size=self.chunk_size,+ chunk_overlap=self.chunk_overlap,+ max_workers=self.max_workers,+ show_progress=self.show_progress,+ )++ def __add__(self, other):+ """Operator overload for '+' to combine knowledge instances.++ Supports multiple combination patterns:+ - ListKnowledge + ListKnowledge → ListKnowledge (merge lists)+ - ListKnowledge + UnitKnowledge → ListKnowledge (append unit to list)++ This enables chain operations like: unit1 + unit2 + unit3++ Args:+ other: Another ListKnowledge or UnitKnowledge with compatible schema.++ Returns:+ New ListKnowledge with combined items.++ Raises:+ TypeError: If schemas don't match or invalid operand type.+ """+ from .unit import UnitKnowledge++ # Case 1: ListKnowledge + ListKnowledge (call parent implementation)+ if isinstance(other, ListKnowledge):+ # Check schema compatibility+ if self.item_schema != other.item_schema:+ raise TypeError(+ f"Cannot add ListKnowledge instances with different schemas. "+ f"Left: {self.item_schema.__name__}, Right: {other.item_schema.__name__}"+ )+ # Use base class merge logic+ return super().__add__(other)++ # Case 2: ListKnowledge + UnitKnowledge → ListKnowledge (append unit)+ elif isinstance(other, UnitKnowledge):+ # Check schema compatibility+ if self.item_schema != other._data_schema:+ raise TypeError(+ f"Cannot add UnitKnowledge to ListKnowledge with different schemas. "+ f"List: {self.item_schema.__name__}, Unit: {other._data_schema.__name__}"+ )++ # Create new ListKnowledge with appended unit+ new_list = self._create_new_instance()+ new_list._data.items = self.items + [other._data]++ # Merge metadata+ new_list.metadata["created_at"] = min(+ self.metadata["created_at"], other.metadata["created_at"]+ )+ new_list.metadata["updated_at"] = datetime.now()++ return new_list++ else:+ raise TypeError(+ f"Unsupported operand type for +: 'ListKnowledge' and '{type(other).__name__}'"+ )++ def extract(self, text: str, *, store: bool = True) -> ItemListSchema:+ """Extracts a list of items using LangChain native batch processing.++ Args:+ text: Input text to extract items from.+ store: Controls whether to store extracted knowledge internally.+ - True (default): Store mode - merge with existing knowledge and update internal state+ - False: Temporary mode - return extracted data without modifying internal state++ Returns:+ The list of extracted items.+ """+ start_time = datetime.now()++ # Determine if chunking is needed+ if len(text) <= self.chunk_size:+ # Short text: extract in one pass+ if self.show_progress:+ logger.info(f"Processing single text (length: {len(text)})...")++ extracted_data = self.llm_chain_extract.invoke({"chunk_text": text})+ extracted_data_list = [extracted_data]++ else:+ # Long text: extract by chunking+ chunks = self.text_splitter.split_text(text)+ logger.info(f"Split text into {len(chunks)} chunks")++ if self.show_progress:+ logger.info(f"Processing {len(chunks)} chunk(s)...")++ inputs = [{"chunk_text": chunk} for chunk in chunks]+ extracted_data_list = self.llm_chain_extract.batch(+ inputs, config={"max_concurrency": self.max_workers}+ )++ if self.show_progress:+ logger.info(f"Extracted {len(extracted_data_list)} chunks")++ logger.info("Merging extracted knowledge...")+ merged_data = self.merge(extracted_data_list)++ # If store=True, merge with existing data and update internal state+ if store:+ if self._data and len(self.items) > 0:+ self._data = self.merge([self._data, merged_data])+ else:+ self._data = merged_data++ self.clear_index()+ self.metadata["updated_at"] = datetime.now()++ logger.info("Knowledge extraction completed")+ logger.info(+ f"Duration: {(datetime.now() - start_time).total_seconds():.2f} seconds"+ )++ # Return items list (either from stored data or temporary merged data)+ return self.items if store else merged_data.items++ def merge(self, data_list: List[ItemListSchema]) -> ItemListSchema:+ """Pure data merge method implementing list append strategy.++ Merge strategy: Collects all items from all container objects and merges them+ into a single list. Subclasses can override this method to implement more+ sophisticated deduplication logic (e.g., EntityKnowledge).++ Args:+ data_list: List of container objects to merge.++ Returns:+ A new merged ItemListSchema object.+ """+ all_items = []++ # Collect all items from each container+ for data in data_list:+ copied_data = data.model_copy(deep=True)+ all_items.extend(copied_data.items)++ # Return a new container object+ return self.item_list_schema(items=all_items)++ def build_index(self):+ """Builds independent vector index for each item in the list."""+ items = self.items+ if not items:+ logger.warning("No items to index")+ return++ if self._index is not None:+ return++ documents = []+ for idx, item in enumerate(items):+ # Serialize each Item as a Document+ content = item.model_dump_json() # Use JSON string as content+ documents.append(+ Document(+ page_content=content,+ metadata={"raw": item.model_dump(), "index": idx},+ )+ )++ if documents:+ try:+ self._index = FAISS.from_documents(documents, self.embedder)+ logger.info(f"Built FAISS index with {len(documents)} items")+ except ImportError:+ logger.error("FAISS not available. Install with: pip install faiss-cpu")+ raise++ def search(self, query: str, top_k: int = 3) -> List[Any]:+ """Searches items in the list using semantic similarity.++ Args:+ query: Search query string.+ top_k: Number of results to return.++ Returns:+ List of relevant items.+ """+ if not self.items:+ logger.warning("No items to search")+ return []++ if self._index is None:+ raise Exception("Vector store not initialized")++ docs = self._index.similarity_search(query, k=top_k)+ results = []+ for doc in docs:+ # Attempt to restore as object+ try:+ raw = doc.metadata.get("raw", {})+ item = self.item_schema.model_validate(raw)+ results.append(item)+ except Exception as e:+ logger.warning(f"Failed to restore item: {e}")+ results.append(doc.metadata.get("raw"))++ logger.info(f"Found {len(results)} results for query: {query[:50]}...")+ return results++ def __len__(self) -> int:+ """Returns the number of elements in the list."""+ return len(self.items)++ def dump(self, folder_path: str) -> Any:+ """Exports knowledge to a specified folder.++ Uses the same serialization logic as UnitKnowledge since the container+ is also a BaseModel.+ """+ folder = Path(folder_path)+ folder.mkdir(parents=True, exist_ok=True)++ # 1. Save structured data+ data = {+ "schema_name": self.item_list_schema.__name__,+ "item_schema_name": self.item_schema.__name__,+ "item_schema": self.item_schema.model_json_schema(),+ "data": self.data.model_dump(),+ "metadata": {+ k: str(v) if isinstance(v, datetime) else v+ for k, v in self.metadata.items()+ },+ }++ json_str = json.dumps(data, indent=2, ensure_ascii=False, default=str)+ data_file = folder / "state.json"+ with open(data_file, "w", encoding="utf-8") as f:+ f.write(json_str)+ logger.info(f"Saved data to {data_file}")++ # 2. Save vector index+ if self._index is not None:+ index_path = str(folder / "faiss_index")+ self._index.save_local(index_path)+ logger.info(f"Saved FAISS index to {index_path}")+ else:+ logger.warning("No index to save")++ return json_str++ def load(self, folder_path: str):+ """Loads knowledge from a specified folder."""+ folder = Path(folder_path)+ if not folder.is_dir():+ raise ValueError(f"Folder does not exist: {folder_path}")++ # 1. Load structured data+ data_file = folder / "state.json"+ if not data_file.is_file():+ raise ValueError(f"Data file not found: {data_file}")++ with open(data_file, "r", encoding="utf-8") as f:+ data = json.load(f)+ logger.info(f"Loaded data from {data_file}")++ self._data = self.item_list_schema.model_validate(data.get("data", {}))++ # Update metadata with loaded values+ if "metadata" in data:+ self.metadata.update(data["metadata"])+ self.metadata["updated_at"] = datetime.now()++ # 2. Load vector index+ index_path = str(folder / "faiss_index")+ if Path(index_path).exists():+ try:+ self._index = FAISS.load_local(+ index_path, self.embedder, allow_dangerous_deserialization=True+ )+ logger.info(f"Loaded FAISS index from {index_path}")+ except Exception as e:+ logger.warning(f"Failed to load FAISS index: {e}")+ self._index = None+ else:+ logger.warning("No index file found, will rebuild on next search")+ self._index = None++ logger.info(f"Loaded knowledge successfully with {len(self)} items")diff --git a/hyperextract/knowledge/generic/set.py b/hyperextract/knowledge/generic/set.pynew file mode 100644index 0000000..c7e861f--- /dev/null+++ b/hyperextract/knowledge/generic/set.py@@ -0,0 +1,18 @@+"""Set Knowledge Pattern - extracts a unique collection of objects from text.++TODO: Implement SetKnowledge class with the following features:+ - Automatic deduplication based on content hash or embedding similarity+ - Set-based merge operations (union, intersection, difference)+ - Efficient membership testing++Example usage:+ class Keyword(BaseModel):+ term: str+ category: str = ""++ knowledge = SetKnowledge(Keyword, llm, embedder)+ knowledge.extract(text) # Automatically deduplicates+"""++# TODO: Implement SetKnowledge[Item] class+# Consider inheritance: SetKnowledge(ListKnowledge[Item]) or SetKnowledge(BaseKnowledge[...])diff --git a/hyperextract/knowledge/generic/unit.py b/hyperextract/knowledge/generic/unit.pynew file mode 100644index 0000000..0e2bae0--- /dev/null+++ b/hyperextract/knowledge/generic/unit.py@@ -0,0 +1,421 @@+"""Unit Knowledge Pattern - extracts a single structured object from text."""++import json+from typing import List, Any, Type+from datetime import datetime+from pathlib import Path+from langchain_core.language_models.chat_models import BaseChatModel+from langchain_core.embeddings import Embeddings+from langchain_core.documents import Document+from langchain_community.vectorstores import FAISS++from ..base import BaseKnowledge, T++try:+ from hyperextract.config import logger+except ImportError:+ import logging+ logger = logging.getLogger(__name__)+++class UnitKnowledge(BaseKnowledge[T]):+ """Unit Knowledge Pattern - extracts a single structured object from text.++ This pattern is designed for extracting exactly one structured object per document,+ regardless of document length. Suitable for document-level information like summaries,+ metadata, or aggregate statistics.++ Key characteristics:+ - Extraction target: One unique structured object per document+ - Merge strategy: Field-level update (Upsert/Patch) - first extraction wins,+ subsequent extractions only fill missing fields+ - Indexing strategy: Each non-null field of the object is indexed independently+ - Processing: Uses LangChain native batch processing for efficient multi-chunk handling+ """++ def __init__(+ self,+ data_schema: Type[T],+ llm_client: BaseChatModel,+ embedder: Embeddings,+ *,+ prompt: str = "",+ chunk_size: int = 2000,+ chunk_overlap: int = 200,+ max_workers: int = 10,+ show_progress: bool = True,+ **kwargs,+ ):+ """Initialize UnitKnowledge with schema and configuration.++ Args:+ data_schema: Pydantic BaseModel subclass defining the object structure.+ llm_client: Language model client for extraction.+ embedder: Embedding model for vector indexing.+ prompt: Custom extraction prompt (defaults to generic prompt).+ chunk_size: Maximum characters per chunk for long texts.+ chunk_overlap: Overlapping characters between adjacent chunks.+ max_workers: Maximum concurrent extraction tasks.+ show_progress: Whether to log progress information.+ """+ super().__init__(+ data_schema,+ llm_client,+ embedder,+ prompt=prompt or self._default_prompt(),+ chunk_size=chunk_size,+ chunk_overlap=chunk_overlap,+ max_workers=max_workers,+ show_progress=show_progress,+ **kwargs,+ )++ @staticmethod+ def _default_prompt() -> str:+ """Returns the default extraction prompt for single-object extraction."""+ return (+ "You are an expert knowledge extraction assistant. "+ "Your task is to carefully analyze the following text and extract structured information "+ "according to the specified schema. Be precise, comprehensive, and faithful to the source text. "+ "Extract all relevant details without adding information not present in the text.\n\n"+ "### Source Text:\n"+ )++ # ==================== Extraction & Aggregation ====================++ def extract(self, text: str, *, store: bool = True) -> T:+ """Extracts knowledge using LangChain native batch processing.++ Automatically handles text chunking for long documents and aggregates results+ using the field-level merge strategy.++ Args:+ text: Input text to extract knowledge from.+ store: Controls whether to store extracted knowledge internally.+ - True (default): Store mode - merge with existing knowledge and update internal state+ - False: Temporary mode - return extracted data without modifying internal state++ Returns:+ The extracted knowledge object.+ """+ start_time = datetime.now()++ # Determine if chunking is needed based on text length+ if len(text) <= self.chunk_size:+ # Short text: direct extraction without chunking+ if self.show_progress:+ logger.info(f"Processing single text (length: {len(text)})...")++ extracted_data = self.llm_chain_extract.invoke({"chunk_text": text})+ extracted_data_list = [extracted_data]++ else:+ # Long text: extract by chunking+ chunks = self.text_splitter.split_text(text)+ logger.info(f"Split text into {len(chunks)} chunks")++ if self.show_progress:+ logger.info(f"Processing {len(chunks)} chunk(s)...")++ inputs = [{"chunk_text": chunk} for chunk in chunks]+ extracted_data_list = self.llm_chain_extract.batch(+ inputs, config={"max_concurrency": self.max_workers}+ )++ if self.show_progress:+ logger.info(f"Extracted {len(extracted_data_list)} chunks")++ logger.info("Merging extracted knowledge...")+ merged_data = self.merge(extracted_data_list)++ # If store=True, merge with existing data and update internal state+ if store:+ if self._data and any(v is not None for v in self._data.model_dump().values()):+ self._data = self.merge([self._data, merged_data])+ else:+ self._data = merged_data++ self.clear_index()+ self.metadata["updated_at"] = datetime.now()++ logger.info("Knowledge extraction completed")+ logger.info(+ f"Duration: {(datetime.now() - start_time).total_seconds():.2f} seconds"+ )++ return self._data if store else merged_data++ def merge(self, data_list: List[T]) -> T:+ """Pure data merge method implementing field-level update strategy.++ Merge strategy: First extraction takes precedence. Subsequent extractions only fill+ missing fields without overwriting existing values. Implemented using model_copy(update=...).++ Args:+ data_list: List of extracted data objects to merge.++ Returns:+ A new merged knowledge object.+ """+ result = data_list[0].model_copy()++ for item in data_list[1:]:+ result = item.model_copy(update=result.model_dump(exclude_none=True))++ return result++ def __add__(self, other):+ """Operator overload for '+' to combine UnitKnowledge instances into ListKnowledge.++ When two UnitKnowledge instances are added, they are combined into a ListKnowledge+ containing both objects as separate items. This enables intuitive collection building.++ Usage:+ >>> unit1 = UnitKnowledge(PersonSchema, ...)+ >>> unit2 = UnitKnowledge(PersonSchema, ...)+ >>> person_list = unit1 + unit2 # → ListKnowledge[PersonSchema]+ >>>+ >>> # Chain operations+ >>> unit3 = UnitKnowledge(PersonSchema, ...)+ >>> person_list = unit1 + unit2 + unit3 # → ListKnowledge with 3 items++ Args:+ other: Another UnitKnowledge with the same data schema, or ListKnowledge.++ Returns:+ ListKnowledge containing both objects as items.++ Raises:+ TypeError: If schemas don't match or invalid operand type.+ """+ from .list import ListKnowledge++ # Case 1: UnitKnowledge + UnitKnowledge → ListKnowledge+ if isinstance(other, UnitKnowledge):+ # Check schema compatibility+ if self._data_schema != other._data_schema:+ raise TypeError(+ f"Cannot add UnitKnowledge instances with different schemas. "+ f"Left: {self._data_schema.__name__}, Right: {other._data_schema.__name__}"+ )++ # Create new ListKnowledge+ list_kb = ListKnowledge(+ item_schema=self._data_schema,+ llm_client=self.llm_client,+ embedder=self.embedder,+ prompt=self.prompt,+ chunk_size=self.chunk_size,+ chunk_overlap=self.chunk_overlap,+ max_workers=self.max_workers,+ show_progress=self.show_progress,+ )++ # Set items from both units+ list_kb._data.items = [self._data, other._data]++ # Merge metadata+ list_kb.metadata["created_at"] = min(+ self.metadata["created_at"], other.metadata["created_at"]+ )+ list_kb.metadata["updated_at"] = datetime.now()++ return list_kb++ # Case 2: UnitKnowledge + ListKnowledge → ListKnowledge (for reverse order)+ elif isinstance(other, ListKnowledge):+ # Check schema compatibility+ if self._data_schema != other.item_schema:+ raise TypeError(+ f"Cannot add UnitKnowledge to ListKnowledge with different schemas. "+ f"Unit: {self._data_schema.__name__}, List: {other.item_schema.__name__}"+ )++ # Create new ListKnowledge with unit prepended+ new_list = ListKnowledge(+ item_schema=self._data_schema,+ llm_client=self.llm_client,+ embedder=self.embedder,+ prompt=self.prompt,+ chunk_size=self.chunk_size,+ chunk_overlap=self.chunk_overlap,+ max_workers=self.max_workers,+ show_progress=self.show_progress,+ )++ # Prepend self to other's items+ new_list._data.items = [self._data] + other.items++ # Merge metadata+ new_list.metadata["created_at"] = min(+ self.metadata["created_at"], other.metadata["created_at"]+ )+ new_list.metadata["updated_at"] = datetime.now()++ return new_list++ else:+ raise TypeError(+ f"Unsupported operand type for +: 'UnitKnowledge' and '{type(other).__name__}'"+ )++ # ==================== Indexing & Query ====================++ def build_index(self):+ """Builds vector index from all non-null fields in the data object."""+ # Check if there's data to index+ if len(self) == 0:+ logger.warning("No data to index")+ return++ if self._index is not None:+ return++ documents = []+ for field_name in self.data_schema.model_fields:+ field_value = getattr(self._data, field_name)+ if field_value is not None:+ documents.append(+ Document(+ page_content=field_name,+ metadata={"raw": {field_name: field_value}},+ )+ )++ self._index = FAISS.from_documents(documents, self.embedder)+ logger.info(f"Built FAISS index with {len(documents)} documents")++ def search(self, query: str, top_k: int = 3) -> List[Any]:+ """Searches all indexed fields using semantic similarity.++ Args:+ query: Search query string.+ top_k: Number of results to return.++ Returns:+ List of relevant knowledge items (field-value dictionaries).+ """+ # Check if there's data to search+ if len(self) == 0:+ logger.warning("No items to search")+ return []++ if self._index is None:+ raise Exception("Index is not built, please build the index first.")++ docs = self._index.similarity_search(query, k=top_k)++ # Restore original objects from metadata+ results = []+ for doc in docs:+ try:+ raw_data = doc.metadata["raw"] # {field_name: field_value}+ results.append(raw_data)+ except Exception as e:+ logger.warning(f"Failed to restore item: {e}")++ logger.info(f"Found {len(results)} results for query: {query[:50]}...")+ return results++ def __len__(self) -> int:+ """Returns 1 since UnitKnowledge represents a single knowledge unit."""+ return 1++ # ==================== Serialization ====================++ def dump(self, folder_path: str | Path):+ """Exports knowledge to a specified folder.++ Saves to the folder:+ 1. Structured data (self._data) - saved as state.json+ 2. Vector index (self._index) - saved as FAISS index files++ Args:+ folder_path: Target folder path for saving.+ """++ folder = Path(folder_path)+ if folder.exists() and folder.is_file():+ raise Exception("Folder path is a file, please provide a folder path.")++ if not folder.exists():+ logger.info(f"Creating folder: {folder}")+ folder.mkdir(parents=True, exist_ok=True)++ try:+ # 1. Save structured data+ data = {+ "schema_name": self.data_schema.__name__,+ "data_schema": self.data_schema.model_json_schema(),+ "data": self.data.model_dump(),+ "metadata": {+ k: str(v) if isinstance(v, datetime) else v+ for k, v in self.metadata.items()+ },+ }++ json_str = json.dumps(data, indent=2, ensure_ascii=False, default=str)+ data_file = folder / "state.json"+ with open(data_file, "w", encoding="utf-8") as f:+ f.write(json_str)+ logger.info(f"Saved data to {data_file}")++ # 2. Save vector index+ if self._index is not None:+ index_path = str(folder / "faiss_index")+ self._index.save_local(index_path)+ logger.info(f"Saved FAISS index to {index_path}")+ else:+ logger.warning("No index to save")++ except Exception as e:+ logger.error(f"Failed to dump knowledge: {e}")++ def load(self, folder_path: str | Path):+ """Loads knowledge from a specified folder.++ Loads from the folder:+ 1. Structured data (self._data) - loaded from state.json+ 2. Vector index (self._index) - loaded from FAISS index files++ Args:+ folder_path: Source folder path containing saved knowledge.+ """+ folder = Path(folder_path)+ if not folder.is_dir():+ raise ValueError(f"Folder does not exist: {folder_path}")++ # 1. Load structured data+ data_file = folder / "state.json"+ if not data_file.is_file():+ raise ValueError(f"Data file not found: {data_file}")++ with open(data_file, "r", encoding="utf-8") as f:+ data = json.load(f)+ logger.info(f"Loaded data from {data_file}")++ self._data = self.data_schema.model_validate(data.get("data", {}))++ # Update metadata with loaded values+ if "metadata" in data:+ self.metadata.update(data["metadata"])+ self.metadata["updated_at"] = datetime.now()++ # 2. Load vector index+ index_path = str(folder / "faiss_index")+ if Path(index_path).exists():+ try:+ from langchain_community.vectorstores import FAISS++ self._index = FAISS.load_local(+ index_path, self.embedder, allow_dangerous_deserialization=True+ )+ logger.info(f"Loaded FAISS index from {index_path}")+ except Exception as e:+ logger.warning(f"Failed to load FAISS index: {e}")+ self._index = None+ else:+ logger.warning("No index file found, will rebuild on next search")+ self._index = None++ logger.info(f"Loaded knowledge successfully with {len(self)} unit(s)")diff --git a/src/knowledge/graph/__init__.py b/hyperextract/knowledge/graph/__init__.pysimilarity index 100%rename from src/knowledge/graph/__init__.pyrename to hyperextract/knowledge/graph/__init__.pydiff --git a/src/knowledge/graph/base.py b/hyperextract/knowledge/graph/base.pysimilarity index 100%rename from src/knowledge/graph/base.pyrename to hyperextract/knowledge/graph/base.pydiff --git a/src/knowledge/graph/kg_gen.py b/hyperextract/knowledge/graph/kg_gen.pysimilarity index 100%rename from src/knowledge/graph/kg_gen.pyrename to hyperextract/knowledge/graph/kg_gen.pydiff --git a/src/knowledge/hypergraph/__init__.py b/hyperextract/knowledge/hypergraph/__init__.pysimilarity index 100%rename from src/knowledge/hypergraph/__init__.pyrename to hyperextract/knowledge/hypergraph/__init__.pydiff --git a/src/knowledge/hypergraph/base.py b/hyperextract/knowledge/hypergraph/base.pysimilarity index 100%rename from src/knowledge/hypergraph/base.pyrename to hyperextract/knowledge/hypergraph/base.pydiff --git a/src/knowledge/hypergraph/hyperrag.py b/hyperextract/knowledge/hypergraph/hyperrag.pysimilarity index 100%rename from src/knowledge/hypergraph/hyperrag.pyrename to hyperextract/knowledge/hypergraph/hyperrag.pydiff --git a/hyperextract/knowledge/typical/__init__.py b/hyperextract/knowledge/typical/__init__.pynew file mode 100644index 0000000..2daeea9--- /dev/null+++ b/hyperextract/knowledge/typical/__init__.py@@ -0,0 +1,7 @@+"""Typical knowledge patterns - common knowledge representation types."""++from .entity import EntityKnowledge++__all__ = [+ "EntityKnowledge",+]diff --git a/src/knowledge/entity.py b/hyperextract/knowledge/typical/entity.pysimilarity index 92%rename from src/knowledge/entity.pyrename to hyperextract/knowledge/typical/entity.pyindex e692764..6189e06 100644--- a/src/knowledge/entity.py+++ b/hyperextract/knowledge/typical/entity.py@@ -11,10 +11,10 @@ from langchain_core.documents import Documentfrom langchain_core.prompts import ChatPromptTemplatefrom langchain_community.vectorstores import FAISS-from .generic import ListKnowledge+from ..generic import ListKnowledgetry:- from src.config import logger+ from hyperextract.config import loggerexcept ImportError:import logging@@ -127,34 +127,37 @@ Please merge these two entities intelligently and return the result."""# ==================== Extraction & Aggregation ====================- def extract(self, text: str, *, merge_mode: bool = False) -> BaseModel:- """Extracts entity knowledge with support for replace/accumulate modes.+ def extract(self, text: str, *, store: bool = True) -> BaseModel:+ """Extracts entity knowledge with support for store/temporary modes.Mode explanation:- - merge_mode=False (default): Replace mode- Only uses newly extracted entities+ - store=True (default): Store mode+ Merges newly extracted entities with existing knowledge and updates internal state- - merge_mode=True: Accumulate mode- Keeps existing entities, extracts new entities and intelligently merges- (LLM-powered deduplication)+ - store=False: Temporary mode+ Returns extracted entities without modifying internal stateArgs:text: Input text to extract entities from.- merge_mode: Merge mode (default False).+ store: Whether to store extracted knowledge (default True).Returns:Container with extracted entities."""- # Call parent extraction method with merge_mode- result = super().extract(text, merge_mode=merge_mode)+ # Call parent extraction method with store parameter+ result = super().extract(text, store=store)- # Synchronize extracted items to entity_map after extraction- self._sync_items_to_map()-- logger.info(- f"Entity extraction completed ({'Accumulate' if merge_mode else 'Replace'} mode)"- )- logger.info(f"Total entities: {len(self._entity_map)}")+ # Only synchronize to entity_map if storing internally+ if store:+ self._sync_items_to_map()+ logger.info(+ "Entity extraction completed (Store mode)"+ )+ logger.info(f"Total entities: {len(self._entity_map)}")+ else:+ logger.info(+ "Entity extraction completed (Temporary mode)"+ )return resultdiff --git a/src/knowledge/relation.py b/hyperextract/knowledge/typical/relation.pysimilarity index 100%rename from src/knowledge/relation.pyrename to hyperextract/knowledge/typical/relation.pydiff --git a/src/skill/__init__.py b/hyperextract/knowledge/typical/tree.pysimilarity index 100%rename from src/skill/__init__.pyrename to hyperextract/knowledge/typical/tree.pydiff --git a/hyperextract/skill/__init__.py b/hyperextract/skill/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/pyproject.toml b/pyproject.tomlindex f8c124a..c0970ff 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,5 +1,5 @@[project]-name = "hyper-extract"+name = "hyperextract"version = "0.1.0"description = "An intelligent, LLM-powered knowledge extraction and evolution framework with semantic search capabilities"readme = "README.md"diff --git a/src/knowledge/generic.py b/src/knowledge/generic.pydeleted file mode 100644index 6f50533..0000000--- a/src/knowledge/generic.py+++ /dev/null@@ -1,665 +0,0 @@-"""Fundamental knowledge pattern implementations.--This module provides two core knowledge patterns:- - UnitKnowledge: Single-object pattern for extracting one structured object per document- (suitable for summaries, metadata, etc.)- - ListKnowledge: Collection pattern for extracting multiple items from text- (suitable for entities, events, references, etc.)-"""--import json-from typing import List, Any, Type, TypeVar, Generic, TYPE_CHECKING-from datetime import datetime-from pathlib import Path-from pydantic import BaseModel, Field, create_model-from langchain_core.language_models.chat_models import BaseChatModel-from langchain_core.embeddings import Embeddings-from langchain_core.documents import Document-from langchain_community.vectorstores import FAISS--from .base import BaseKnowledge, T--try:- from src.config import logger-except ImportError:- import logging-- logger = logging.getLogger(__name__)--# ==================== Two Fundamental Knowledge Patterns ====================--# UnitKnowledge: Single-object knowledge pattern---class UnitKnowledge(BaseKnowledge[T]):- """Unit Knowledge Pattern - extracts a single structured object from text.-- This pattern is designed for extracting exactly one structured object per document,- regardless of document length. Suitable for document-level information like summaries,- metadata, or aggregate statistics.-- Key characteristics:- - Extraction target: One unique structured object per document- - Merge strategy: Field-level update (Upsert/Patch) - first extraction wins,- subsequent extractions only fill missing fields- - Indexing strategy: Each non-null field of the object is indexed independently- - Processing: Uses LangChain native batch processing for efficient multi-chunk handling- """-- def __init__(- self,- data_schema: Type[T],- llm_client: BaseChatModel,- embedder: Embeddings,- *,- prompt: str = "",- chunk_size: int = 2000,- chunk_overlap: int = 200,- max_workers: int = 10,- show_progress: bool = True,- **kwargs,- ):- """Initialize UnitKnowledge with schema and configuration.-- Args:- data_schema: Pydantic BaseModel subclass defining the object structure.- llm_client: Language model client for extraction.- embedder: Embedding model for vector indexing.- prompt: Custom extraction prompt (defaults to generic prompt).- chunk_size: Maximum characters per chunk for long texts.- chunk_overlap: Overlapping characters between adjacent chunks.- max_workers: Maximum concurrent extraction tasks.- show_progress: Whether to log progress information.- """- super().__init__(- data_schema,- llm_client,- embedder,- prompt=prompt or self._default_prompt(),- chunk_size=chunk_size,- chunk_overlap=chunk_overlap,- max_workers=max_workers,- show_progress=show_progress,- **kwargs,- )-- @staticmethod- def _default_prompt() -> str:- """Returns the default extraction prompt for single-object extraction."""- return (- "You are an expert knowledge extraction assistant. "- "Your task is to carefully analyze the following text and extract structured information "- "according to the specified schema. Be precise, comprehensive, and faithful to the source text. "- "Extract all relevant details without adding information not present in the text.\n\n"- "### Source Text:\n"- )-- # ==================== Extraction & Aggregation ====================-- def extract(self, text: str, *, merge_mode: bool = False) -> T:- """Extracts knowledge using LangChain native batch processing.-- Automatically handles text chunking for long documents and aggregates results- using the field-level merge strategy.-- Args:- text: Input text to extract knowledge from.- merge_mode: Controls data combination behavior.- - False (default): Replace mode - clears existing data before extraction- - True: Accumulative mode - merges new data with existing knowledge-- Returns:- The extracted knowledge object.- """- start_time = datetime.now()-- # In replace mode, clear existing data first- if not merge_mode:- self.clear()-- # Determine if chunking is needed based on text length- if len(text) <= self.chunk_size:- # Short text: direct extraction without chunking- if self.show_progress:- logger.info(f"Processing single text (length: {len(text)})...")-- extracted_data = self.llm_chain_extract.invoke({"chunk_text": text})- extracted_data_list = [extracted_data]-- else:- # Long text: extract by chunking- chunks = self.text_splitter.split_text(text)- logger.info(f"Split text into {len(chunks)} chunks")-- if self.show_progress:- logger.info(f"Processing {len(chunks)} chunk(s)...")-- inputs = [{"chunk_text": chunk} for chunk in chunks]- extracted_data_list = self.llm_chain_extract.batch(- inputs, config={"max_concurrency": self.max_workers}- )-- if self.show_progress:- logger.info(f"Extracted {len(extracted_data_list)} chunks")-- logger.info("Merging extracted knowledge...")- if merge_mode:- extracted_data_list.insert(0, self.data)-- self._data = self.merge(extracted_data_list)-- # Update state uniformly- self.clear_index()- self.metadata["updated_at"] = datetime.now()-- logger.info("Knowledge extraction completed")- logger.info(- f"Duration: {(datetime.now() - start_time).total_seconds():.2f} seconds"- )- return self.data-- def merge(self, data_list: List[T]) -> T:- """Pure data merge method implementing field-level update strategy.-- Merge strategy: First extraction takes precedence. Subsequent extractions only fill- missing fields without overwriting existing values. Implemented using model_copy(update=...).-- Args:- data_list: List of extracted data objects to merge.-- Returns:- A new merged knowledge object.- """- result = data_list[0].model_copy()-- for item in data_list[1:]:- result = item.model_copy(update=result.model_dump(exclude_none=True))-- return result-- # ==================== Indexing & Query ====================-- def build_index(self):- """Builds vector index from all non-null fields in the data object."""- # Check if there's data to index- if len(self) == 0:- logger.warning("No data to index")- return-- if self._index is not None:- return-- documents = []- for field_name in self.data_schema.model_fields:- field_value = getattr(self._data, field_name)- if field_value is not None:- documents.append(- Document(- page_content=field_name,- metadata={"raw": {field_name: field_value}},- )- )-- self._index = FAISS.from_documents(documents, self.embedder)- logger.info(f"Built FAISS index with {len(documents)} documents")-- def search(self, query: str, top_k: int = 3) -> List[Any]:- """Searches all indexed fields using semantic similarity.-- Args:- query: Search query string.- top_k: Number of results to return.-- Returns:- List of relevant knowledge items (field-value dictionaries).- """- # Check if there's data to search- if len(self) == 0:- logger.warning("No items to search")- return []-- if self._index is None:- raise Exception("Index is not built, please build the index first.")-- docs = self._index.similarity_search(query, k=top_k)-- # Restore original objects from metadata- results = []- for doc in docs:- try:- raw_data = doc.metadata["raw"] # {field_name: field_value}- results.append(raw_data)- except Exception as e:- logger.warning(f"Failed to restore item: {e}")-- logger.info(f"Found {len(results)} results for query: {query[:50]}...")- return results-- def __len__(self) -> int:- """Returns 1 since UnitKnowledge represents a single knowledge unit."""- return 1-- # ==================== Serialization ====================-- def dump(self, folder_path: str | Path):- """Exports knowledge to a specified folder.-- Saves to the folder:- 1. Structured data (self._data) - saved as state.json- 2. Vector index (self._index) - saved as FAISS index files-- Args:- folder_path: Target folder path for saving.- """-- folder = Path(folder_path)- if folder.exists() and folder.is_file():- raise Exception("Folder path is a file, please provide a folder path.")-- if not folder.exists():- logger.info(f"Creating folder: {folder}")- folder.mkdir(parents=True, exist_ok=True)-- try:- # 1. Save structured data- data = {- "schema_name": self.data_schema.__name__,- "data_schema": self.data_schema.model_json_schema(),- "data": self.data.model_dump(),- "metadata": {- k: str(v) if isinstance(v, datetime) else v- for k, v in self.metadata.items()- },- }-- json_str = json.dumps(data, indent=2, ensure_ascii=False, default=str)- data_file = folder / "state.json"- with open(data_file, "w", encoding="utf-8") as f:- f.write(json_str)- logger.info(f"Saved data to {data_file}")-- # 2. Save vector index- if self._index is not None:- index_path = str(folder / "faiss_index")- self._index.save_local(index_path)- logger.info(f"Saved FAISS index to {index_path}")- else:- logger.warning("No index to save")-- except Exception as e:- logger.error(f"Failed to dump knowledge: {e}")-- def load(self, folder_path: str | Path):- """Loads knowledge from a specified folder.-- Loads from the folder:- 1. Structured data (self._data) - loaded from state.json- 2. Vector index (self._index) - loaded from FAISS index files-- Args:- folder_path: Source folder path containing saved knowledge.- """- folder = Path(folder_path)- if not folder.is_dir():- raise ValueError(f"Folder does not exist: {folder_path}")-- # 1. Load structured data- data_file = folder / "state.json"- if not data_file.is_file():- raise ValueError(f"Data file not found: {data_file}")-- with open(data_file, "r", encoding="utf-8") as f:- data = json.load(f)- logger.info(f"Loaded data from {data_file}")-- self._data = self.data_schema.model_validate(data.get("data", {}))-- # Update metadata with loaded values- if "metadata" in data:- self.metadata.update(data["metadata"])- self.metadata["updated_at"] = datetime.now()-- # 2. Load vector index- index_path = str(folder / "faiss_index")- if Path(index_path).exists():- try:- from langchain_community.vectorstores import FAISS-- self._index = FAISS.load_local(- index_path, self.embedder, allow_dangerous_deserialization=True- )- logger.info(f"Loaded FAISS index from {index_path}")- except Exception as e:- logger.warning(f"Failed to load FAISS index: {e}")- self._index = None- else:- logger.warning("No index file found, will rebuild on next search")- self._index = None-- logger.info(f"Loaded knowledge successfully with {len(self)} unit(s)")---# ==================== List Knowledge Pattern ===================--Item = TypeVar("Item", bound=BaseModel)---class ItemListSchema(BaseModel, Generic[Item]):- """Generic schema container for list-based knowledge patterns."""-- items: List[Item] = Field(default_factory=list, description="Item list")---class ListKnowledge(BaseKnowledge[ItemListSchema[Item]], Generic[Item]):- """List Knowledge Pattern - extracts a collection of objects from text.-- This pattern extracts multiple independent objects from a document, suitable for- extracting entities, events, references, or any collection of structured items.-- Key characteristics:- - Extraction target: A collection of structured objects- - Merge strategy: Append with basic deduplication (extensible by subclasses)- - Indexing strategy: Each item in the list is indexed independently-- Comparison with UnitKnowledge:- - UnitKnowledge: Extracts a single structured object (e.g., summary, metadata)- - ListKnowledge: Extracts multiple independent objects (e.g., entity list, event list)- """-- if TYPE_CHECKING:- # Use generic version during type checking to maintain complete type hints- item_list_schema: Type[ItemListSchema[Item]]-- def __init__(- self,- item_schema: Type[Item],- llm_client: BaseChatModel,- embedder: Embeddings,- *,- prompt: str = "",- chunk_size: int = 2000,- chunk_overlap: int = 200,- max_workers: int = 10,- show_progress: bool = True,- **kwargs,- ):- """Initialize ListKnowledge with item schema and configuration.-- Args:- item_schema: Pydantic BaseModel subclass for individual list items.- llm_client: Language model client for extraction.- embedder: Embedding model for vector indexing.- prompt: Custom extraction prompt (defaults to list-oriented prompt).- chunk_size: Maximum characters per chunk for long texts.- chunk_overlap: Overlapping characters between adjacent chunks.- max_workers: Maximum concurrent extraction tasks.- show_progress: Whether to log progress information.- """- self.item_schema = item_schema-- container_name = f"{item_schema.__name__}List"- self.item_list_schema = create_model(- container_name,- items=(- List[item_schema],- Field(default_factory=list, description="Item list"),- ),- )-- super().__init__(- data_schema=self.item_list_schema,- llm_client=llm_client,- embedder=embedder,- prompt=prompt or self._default_prompt(),- chunk_size=chunk_size,- chunk_overlap=chunk_overlap,- max_workers=max_workers,- show_progress=show_progress,- **kwargs,- )-- @staticmethod- def _default_prompt() -> str:- """Returns the default extraction prompt for list-based extraction."""- return (- "You are an expert knowledge extraction assistant. "- "Extract all relevant items from the text into a list. "- "Be comprehensive and ensure no item is missed. "- "Extract all items without adding information not present in the text.\n\n"- "### Source Text:\n"- )-- @property- def items(self) -> List[Item]:- """Returns the internal list of extracted items."""- return getattr(self._data, "items", [])-- def extract(self, text: str, *, merge_mode: bool = False) -> ItemListSchema:- """Extracts a list of items using LangChain native batch processing.-- Args:- text: Input text to extract items from.- merge_mode: Controls data combination behavior.- - False (default): Replace mode - clears existing data before extraction- - True: Accumulative mode - merges new items with existing collection-- Returns:- The list of extracted items.- """- start_time = datetime.now()-- # Replace mode: clear first- if not merge_mode:- self.clear()-- # Determine if chunking is needed- if len(text) <= self.chunk_size:- # Short text: extract in one pass- if self.show_progress:- logger.info(f"Processing single text (length: {len(text)})...")-- extracted_data = self.llm_chain_extract.invoke({"chunk_text": text})- extracted_data_list = [extracted_data]-- else:- # Long text: extract by chunking- chunks = self.text_splitter.split_text(text)- logger.info(f"Split text into {len(chunks)} chunks")-- if self.show_progress:- logger.info(f"Processing {len(chunks)} chunk(s)...")-- inputs = [{"chunk_text": chunk} for chunk in chunks]- extracted_data_list = self.llm_chain_extract.batch(- inputs, config={"max_concurrency": self.max_workers}- )-- if self.show_progress:- logger.info(f"Extracted {len(extracted_data_list)} chunks")-- logger.info("Merging extracted knowledge...")- if merge_mode:- extracted_data_list.insert(0, self.data)-- self._data = self.merge(extracted_data_list)-- # Update state uniformly- self.clear_index()- self.metadata["updated_at"] = datetime.now()-- logger.info("Knowledge extraction completed")- logger.info(- f"Duration: {(datetime.now() - start_time).total_seconds():.2f} seconds"- )- return self.items-- def merge(self, data_list: List[ItemListSchema]) -> ItemListSchema:- """Pure data merge method implementing list append strategy.-- Merge strategy: Collects all items from all container objects and merges them- into a single list. Subclasses can override this method to implement more- sophisticated deduplication logic (e.g., EntityKnowledge).-- Args:- data_list: List of container objects to merge.-- Returns:- A new merged ItemListSchema object.- """- all_items = []-- # Collect all items from each container- for data in data_list:- copied_data = data.model_copy(deep=True)- all_items.extend(copied_data.items)-- # Return a new container object- return self.item_list_schema(items=all_items)-- def build_index(self):- """Builds independent vector index for each item in the list."""- items = self.items- if not items:- logger.warning("No items to index")- return-- if self._index is not None:- return-- documents = []- for idx, item in enumerate(items):- # Serialize each Item as a Document- content = item.model_dump_json() # Use JSON string as content- documents.append(- Document(- page_content=content,- metadata={"raw": item.model_dump(), "index": idx},- )- )-- if documents:- try:- self._index = FAISS.from_documents(documents, self.embedder)- logger.info(f"Built FAISS index with {len(documents)} items")- except ImportError:- logger.error("FAISS not available. Install with: pip install faiss-cpu")- raise-- def search(self, query: str, top_k: int = 3) -> List[Any]:- """Searches items in the list using semantic similarity.-- Args:- query: Search query string.- top_k: Number of results to return.-- Returns:- List of relevant items.- """- if not self.items:- logger.warning("No items to search")- return []-- if self._index is None:- raise Exception("Vector store not initialized")-- docs = self._index.similarity_search(query, k=top_k)- results = []- for doc in docs:- # Attempt to restore as object- try:- raw = doc.metadata.get("raw", {})- item = self.item_schema.model_validate(raw)- results.append(item)- except Exception as e:- logger.warning(f"Failed to restore item: {e}")- results.append(doc.metadata.get("raw"))-- logger.info(f"Found {len(results)} results for query: {query[:50]}...")- return results-- def __len__(self) -> int:- """Returns the number of elements in the list."""- return len(self.items)-- def dump(self, folder_path: str) -> Any:- """Exports knowledge to a specified folder.-- Uses the same serialization logic as UnitKnowledge since the container- is also a BaseModel.- """- from pathlib import Path-- folder = Path(folder_path)- folder.mkdir(parents=True, exist_ok=True)-- # 1. Save structured data- data = {- "schema_name": self.item_list_schema.__name__,- "item_schema_name": self.item_schema.__name__,- "item_schema": self.item_schema.model_json_schema(),- "data": self.data.model_dump(),- "metadata": {- k: str(v) if isinstance(v, datetime) else v- for k, v in self.metadata.items()- },- }-- json_str = json.dumps(data, indent=2, ensure_ascii=False, default=str)- data_file = folder / "state.json"- with open(data_file, "w", encoding="utf-8") as f:- f.write(json_str)- logger.info(f"Saved data to {data_file}")-- # 2. Save vector index- if self._index is not None:- index_path = str(folder / "faiss_index")- self._index.save_local(index_path)- logger.info(f"Saved FAISS index to {index_path}")- else:- logger.warning("No index to save")-- return json_str-- def load(self, folder_path: str, **kwargs):- """Loads knowledge from a specified folder."""- from pathlib import Path-- folder = Path(folder_path)- if not folder.is_dir():- raise ValueError(f"Folder does not exist: {folder_path}")-- # 1. Load structured data- data_file = folder / "state.json"- if not data_file.is_file():- raise ValueError(f"Data file not found: {data_file}")-- with open(data_file, "r", encoding="utf-8") as f:- data = json.load(f)- logger.info(f"Loaded data from {data_file}")-- self._data = self.item_list_schema.model_validate(data.get("data", {}))-- # Update metadata with loaded values- if "metadata" in data:- self.metadata.update(data["metadata"])- self.metadata["updated_at"] = datetime.now()-- # 2. Load vector index- index_path = str(folder / "faiss_index")- if Path(index_path).exists():- try:- self._index = FAISS.load_local(- index_path, self.embedder, allow_dangerous_deserialization=True- )- logger.info(f"Loaded FAISS index from {index_path}")- except Exception as e:- logger.warning(f"Failed to load FAISS index: {e}")- self._index = None- else:- logger.warning("No index file found, will rebuild on next search")- self._index = None-- logger.info(f"Loaded knowledge successfully with {len(self)} items")---# Set Knowledgediff --git a/tests/test_knowledge.py b/tests/test_knowledge.pyindex bb87eaf..ad82fda 100644--- a/tests/test_knowledge.py+++ b/tests/test_knowledge.py@@ -7,7 +7,7 @@ from pydantic import BaseModel, Fieldfrom unittest.mock import MagicMock, AsyncMockimport json-from src.knowledge.base import Knowledge+from hyperextract.knowledge.base import Knowledge# 定义测试用的 Schema