Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
import json
from abc import ABC, abstractmethod
from datetime import datetime
from pathlib import Path
from typing import Any, Generic, TypeVar
from langchain_core.embeddings import Embeddings
from langchain_core.language_models.chat_models import BaseChatModel
from langchain_core.messages import AIMessage
from langchain_core.prompts import ChatPromptTemplate
from langchain_text_splitters import RecursiveCharacterTextSplitter
from pydantic import BaseModel
from hyperextract.utils.logging import get_logger
logger = get_logger(__name__)
T = TypeVar("T", bound=BaseModel)
# ===================== Knowledge Abstract Class =====================
class BaseAutoType(ABC, Generic[T]):
"""Unified knowledge abstract class integrating extraction, storage, and aggregation.
This abstract base class provides a complete framework for managing structured knowledge
extracted from text. It handles the full lifecycle from extraction to serialization.
Responsibilities:
- Extract structured knowledge from text using LLM
- Automatically handle long text chunking and parallel processing
- Store and aggregate extracted knowledge with configurable merge strategies
- Build and maintain vector indices for semantic search
- Provide semantic search and retrieval capabilities
- Support chat/QA interactions by retrieving relevant context and feeding it to LLM
- Provide serialization and deserialization capabilities
"""
# ==================== Initialization & Configuration ====================
def __init__(
self,
data_schema: type[T],
llm_client: BaseChatModel,
embedder: Embeddings,
*,
prompt: str = "",
chunk_size: int = 2048,
chunk_overlap: int = 256,
max_workers: int = 10,
verbose: bool = False,
on_error: str = "skip",
):
"""Initialize the knowledge object with schema and processing configuration.
Args:
data_schema: Pydantic BaseModel subclass defining the knowledge structure.
llm_client: Language model client for extraction.
embedder: Embedding model for semantic search and similarity computation.
prompt: Custom prompt template for extraction (defaults to generic prompt).
chunk_size: Maximum chunk size for splitting long texts.
chunk_overlap: Number of overlapping characters between chunks.
max_workers: Maximum number of concurrent extraction tasks.
verbose: Whether to display detailed execution logs and progress information.
on_error: Per-chunk extraction failure strategy: ``"skip"`` (default)
records the failure in ``extraction_failures`` and continues with
the surviving chunks; ``"raise"`` aborts the whole feed/parse.
"""
self._data_schema = data_schema
self.llm_client = llm_client
self.embedder = embedder
self.prompt = prompt or self._default_prompt()
self.chunk_size = chunk_size
self.chunk_overlap = chunk_overlap
self.max_workers = max_workers
self.verbose = verbose
if on_error not in ("skip", "raise"):
raise ValueError(f"on_error must be 'skip' or 'raise' (got {on_error!r})")
self.on_error = on_error
# Initialize template
self.prompt_template = ChatPromptTemplate.from_template(self.prompt)
self.data_extractor = (
self.prompt_template
| self.llm_client.with_structured_output(
self._data_schema, method="function_calling"
)
)
# Initialize text splitter for chunking long documents
self.text_splitter = RecursiveCharacterTextSplitter(
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
separators=["\n\n", "\n", "。", "!", "?", ". ", "! ", "? ", " ", ""],
)
# Initialize internal state (calls hook for subclass setup)
self._init_internal_state()
# Internal state storing the extracted knowledge
self.metadata: dict[str, Any] = {
"created_at": datetime.now(),
"updated_at": datetime.now(),
}
# Set by parse()/feed_text() while extraction runs; graph-family
# subclasses record raw extraction results under it (provenance).
self._pending_source_id: str | None = None
# Per-run chunk extraction failures: [{"chunk_index", "stage", "error"}].
# Reset at the start of each feed_text()/parse() run.
self._last_extraction_failures: list[dict[str, Any]] = []
def _create_empty_instance(self) -> "BaseAutoType[T]":
"""Creates a new empty instance with the same configuration as this one.
Subclasses can override this method if they have special initialization requirements.
Returns:
A new empty knowledge instance with the same configuration.
"""
return self.__class__(
data_schema=self._data_schema,
llm_client=self.llm_client,
embedder=self.embedder,
prompt=self.prompt,
chunk_size=self.chunk_size,
chunk_overlap=self.chunk_overlap,
max_workers=self.max_workers,
verbose=self.verbose,
)
@abstractmethod
def _default_prompt(self) -> str:
"""Returns the default extraction prompt template.
Subclasses must implement this to provide a prompt tailored to their extraction pattern.
"""
# ==================== Data Access Interface ====================
@property
def extraction_failures(self) -> list[dict[str, Any]]:
"""Chunk extraction failures of the last feed_text()/parse() run.
One entry per failed chunk: ``{"chunk_index": int, "stage": str,
"error": str}``. Empty when every chunk succeeded (or nothing was
fed yet). Always populated — even with the default ``on_error="skip"``
strategy — so callers can detect silently dropped chunks.
"""
return list(self._last_extraction_failures)
@property
def data_schema(self) -> type[T]:
"""Returns the Pydantic schema class used by this knowledge instance.
Returns:
The Pydantic BaseModel subclass defining the knowledge structure.
"""
return self._data_schema
@property
@abstractmethod
def data(self) -> T:
"""Returns all stored knowledge (read-only access).
This is an abstract property that subclasses must implement.
Subclasses may apply transformations to convert internal _data structure
to the external Schema T if they differ (e.g., AutoSet converts OMem → List).
Returns:
The internal knowledge data as a Pydantic model instance (or converted form).
"""
@abstractmethod
def empty(self) -> bool:
"""Checks if the knowledge abstract is currently empty.
Returns:
True if no data is stored, False otherwise.
"""
# ==================== Data Management Operations ====================
def clear(self):
"""Clears all knowledge including data and vector index."""
self._init_internal_state()
self.metadata["updated_at"] = datetime.now()
def clear_index(self):
"""Clears the vector index without affecting the stored data."""
self._init_index_state()
# ==================== Lifecycle Hooks: State Management ====================
def _init_internal_state(self) -> None:
"""Master control: Initialize or reset all internal state (INIT/RESET).
This concrete method orchestrates the reset process by calling two hooks:
1. _init_data_state() - Subclass responsibility: reset data structures
2. _init_index_state() - Default: reset index, can be overridden by subclass
Called in two scenarios:
- During __init__ to set up the initial state
- When clear() is called to reset to empty state
"""
self._init_data_state()
self._init_index_state()
@abstractmethod
def _init_data_state(self) -> None:
"""HOOK: Initialize or reset data structures to empty state.
Subclass Responsibility:
- Initialize self._data with appropriate structure (may differ from Schema T)
- Reset any auxiliary data structures (e.g., lookup dicts, caches)
Subclasses must implement this to set up internal structures that may be optimized
beyond the standard Pydantic schema (e.g., OMem for AutoSet, dict-based for others).
"""
@abstractmethod
def _set_data_state(self, data: T) -> None:
"""HOOK: Overwrite data state with new data (SET).
Called by parse() or load() where the data provided IS the new state.
Subclass Responsibilities:
1. Replace self._data with new data (full reset)
2. Convert standard Schema T to optimized internal structure if needed
3. Invalidate vector index (self.clear_index())
Args:
data: The new data object to set.
"""
@abstractmethod
def _update_data_state(self, incoming_data: T) -> None:
"""HOOK: Merge new data into state (UPDATE).
Called by feed() where the data provided is INCREMENTAL.
Subclass Responsibilities:
1. Merge incoming_data into current data state (optimized for incremental updates)
2. Invalidate vector index (self.clear_index())
Subclasses should implement optimized incremental updates (e.g., set.add instead of
full merge_batch, graph.add_edge instead of graph rebuild).
Args:
incoming_data: The incremental data to merge into the current state.
"""
@abstractmethod
def _init_index_state(self) -> None:
"""HOOK: Initialize or reset vector index to empty state.
Subclass Responsibility:
- Initialize or reset vector index structures
- Can be FAISS, Chroma, Pinecone, or custom implementation
- Typically sets self._index = None or initializes specific index instance
This separation allows index implementation to be decoupled from base class.
"""
# ==================== Extraction & Merge ====================
def _extract_data(self, text: str) -> T:
"""Internal: Unified extraction logic (Chunking -> LLM -> Merge).
Subclasses that support source attribution read
``self._pending_source_id`` (set by parse/feed_text) and record
their raw, pre-merge results via ``_record_extraction_source``.
"""
logger.debug(
"stage=extract_start input_chars=%d chunk_size=%d",
len(text),
self.chunk_size,
)
if len(text) <= self.chunk_size:
logger.debug("stage=extract_single_chunk chunk_text_preview=%s", text[:200])
extracted_data = self._invoke_safe(
self.data_extractor, {"source_text": text}, stage="extract"
)
logger.debug(
"stage=extract_single_chunk_result chunk=0 result_summary=%s",
self._summarize_extracted(extracted_data),
)
extracted_data_list = [extracted_data]
else:
chunks = self.text_splitter.split_text(text)
logger.debug("stage=text_split num_chunks=%d", len(chunks))
for i, chunk in enumerate(chunks):
logger.debug(
"stage=chunk_before_llm chunk_index=%d chunk_chars=%d chunk_text_preview=%s",
i,
len(chunk),
chunk[:200],
)
inputs = [{"source_text": chunk} for chunk in chunks]
logger.debug(
"stage=llm_batch_start max_concurrency=%d num_inputs=%d",
self.max_workers,
len(inputs),
)
extracted_data_list = self._batch_safe(
self.data_extractor, inputs, stage="extract"
)
logger.debug(
"stage=llm_batch_complete results=%d", len(extracted_data_list)
)
for i, result in enumerate(extracted_data_list):
logger.debug(
"stage=chunk_llm_result chunk_index=%d result_summary=%s",
i,
self._summarize_extracted(result),
)
# Filter out None results from failed LLM extractions
extracted_data_list = self._filter_none_results(extracted_data_list)
logger.debug("stage=merge_start num_items=%d", len(extracted_data_list))
merged_data = self.merge_batch_data(extracted_data_list)
logger.debug(
"stage=extract_complete merged_summary=%s",
self._summarize_extracted(merged_data),
)
return merged_data
def _filter_none_results(self, results: list, default_factory=None) -> list:
"""Replace None results from batch LLM extractions with empty defaults.
When the LLM fails to parse output (e.g., context length exceeded,
json_schema validation failure), batch() returns None for that item.
This method replaces None with empty objects to maintain index alignment
with the original chunks list.
Args:
results: List of extraction results, may contain None values.
default_factory: Callable that creates a default empty object.
If None, None values are simply removed (legacy behavior).
Returns:
List with None values replaced or removed.
"""
if not results:
return results
none_count = sum(1 for r in results if r is None)
if none_count > 0:
logger.warning(
"stage=batch_filter none_results_detected none_count=%d total=%d",
none_count,
len(results),
)
if default_factory is not None:
return [r if r is not None else default_factory() for r in results]
return [r for r in results if r is not None]
return results
def _invoke_safe(self, extractor, input: dict, *, stage: str):
"""invoke() with provider failures logged and nulled instead of raised.
One bad chunk (rate limit, timeout, unparseable output) must not abort
the whole run — failures degrade to None, matching _extract_data's
contract. Logs the stage and exception only, never the source text.
Args:
extractor: Runnable with an ``invoke`` method.
input: Input dict for the extractor.
stage: Stage label used in log lines (e.g. ``"one_stage"``).
Returns:
The extractor result, or None if it raised.
"""
try:
return extractor.invoke(input)
except Exception as e:
logger.warning("stage=%s_single_extract_failed error=%s", stage, e)
self._last_extraction_failures.append(
{"chunk_index": 0, "stage": stage, "error": str(e)}
)
if self.on_error == "raise":
raise
def _batch_safe(self, extractor, inputs: list[dict], *, stage: str) -> list:
"""batch() with return_exceptions=True; per-chunk failures logged and nulled.
Args:
extractor: Runnable with a ``batch`` method.
inputs: List of input dicts, one per chunk.
stage: Stage label used in log lines (e.g. ``"two_stage_nodes"``).
Returns:
List aligned with ``inputs``; failed chunks are None.
"""
raw = extractor.batch(
inputs,
config={"max_concurrency": self.max_workers},
return_exceptions=True,
)
results: list = []
for i, r in enumerate(raw):
if isinstance(r, Exception):
logger.warning(
"stage=%s_chunk_extract_failed chunk_index=%d error=%s", stage, i, r
)
self._last_extraction_failures.append(
{"chunk_index": i, "stage": stage, "error": str(r)}
)
if self.on_error == "raise":
raise r
results.append(None)
else:
results.append(r)
return results
def _adopt_source_ledger(self, other: "BaseAutoType[T]", source_id: str) -> None:
"""Hook: transfer ledger entries for ``source_id`` from ``other`` to
``self`` after parse(). No-op by default; graph-family memories with
``track_sources`` override this."""
return
def _dump_provenance(self, root: Path) -> None:
"""Hook: persist the source ledger alongside the KA. No-op by default."""
return
def _load_provenance(self, root: Path) -> None:
"""Hook: restore the source ledger alongside the KA. No-op by default."""
return
def _summarize_extracted(self, data: T) -> str:
"""Return a concise summary of extracted data for debug logging."""
try:
dump = data.model_dump()
# Count entities and relations for graph-type schemas
entities = len(dump.get("entities", []))
relations = len(dump.get("relations", []))
if entities or relations:
return f"entities={entities} relations={relations}"
# Generic fallback: list top-level keys with their lengths
parts = []
for key, val in dump.items():
if isinstance(val, (list, tuple)):
parts.append(f"{key}={len(val)}")
elif isinstance(val, str):
parts.append(f"{key}={val[:50]!r}")
return ", ".join(parts) if parts else str(dump)[:100]
except Exception:
return repr(data)[:100]
def parse(self, text: str, *, source_id: str | None = None) -> "BaseAutoType[T]":
"""
Parses knowledge into a NEW instance without modifying the current one.
Use this for previewing data or branching knowledge abstracts.
Args:
text: Input text.
source_id: Optional source attribution (graph-family memories with
``track_sources``): the raw extraction results are recorded
under this id so the document's contributions can later be
rolled back via ``remove_source``.
Returns:
A new knowledge instance containing only the parsed data.
"""
self._pending_source_id = source_id
self._last_extraction_failures = []
try:
parsed_data = self._extract_data(text)
finally:
self._pending_source_id = None
new_instance = self._create_empty_instance()
new_instance._set_data_state(parsed_data)
if source_id:
new_instance._adopt_source_ledger(self, source_id)
new_instance.metadata["created_at"] = datetime.now()
new_instance.metadata["updated_at"] = datetime.now()
return new_instance
def feed_text(
self,
text: str,
*,
source_id: str | None = None,
content_hash: str | None = None,
) -> "BaseAutoType[T]":
"""
Ingests text into the CURRENT knowledge abstract instance.
This modifies the internal state by merging new data with existing data.
Supports method chaining (e.g., ka.feed_text(text1).feed_text(text2)).
Args:
text: Input text.
source_id: Optional source attribution (graph-family memories with
``track_sources``): the raw extraction results are recorded
under this id so the document's contributions can later be
rolled back via ``remove_source``.
content_hash: Optional hash of the source content (recorded in
the source ledger for change detection).
Returns:
Self (the current instance).
"""
logger.debug("stage=feed_text_start input_chars=%d", len(text))
self._pending_source_id = source_id
self._pending_content_hash = content_hash
self._last_extraction_failures = []
try:
extracted_data = self._extract_data(text)
logger.debug("stage=extract_done")
# Use UPDATE hook instead of manual merge+set
self._update_data_state(extracted_data)
logger.debug("stage=data_merged")
finally:
self._pending_source_id = None
self._pending_content_hash = None
self.metadata["updated_at"] = datetime.now()
return self
@abstractmethod
def merge_batch_data(self, data_list: list[T]) -> T:
"""Merges multiple knowledge data objects into a single unified object.
This is a pure data transformation method that does not modify internal state.
Subclasses implement specific merge strategies (deduplication, conflict resolution, etc.).
The batch merge is typically used during multi-chunk extraction where results from
different chunks need to be aggregated into a single knowledge object.
Responsibilities:
- Implement concrete merge algorithms (deduplication, conflict resolution, etc.)
- Return a new merged data object
- Never modify instance attributes
Args:
data_list: List of knowledge data objects to merge from batch processing.
Returns:
A new merged knowledge object.
"""
# ==================== Indexing & Search & Chat ====================
@abstractmethod
def build_index(self):
"""Builds or rebuilds the vector index for semantic search.
Subclasses must implement this method to define how the vector index is constructed
from the knowledge data. Uses FAISS as the vector store backend.
"""
@abstractmethod
def search(self, query: str, top_k: int = 3) -> list[Any]:
"""Performs semantic search over the knowledge abstract.
Standard search workflow:
1. Ensure index is built (call build_index if needed)
2. Use vector_store.similarity_search for retrieval
3. Restore original data structures from Document.metadata
Subclasses can override this method to implement custom search logic.
Args:
query: Search query string.
top_k: Number of results to return.
Returns:
List of relevant knowledge items.
"""
def chat(
self,
query: str,
top_k: int = 3,
*,
source_ids: list[str] | None = None,
tags: list[str] | None = None,
) -> AIMessage:
"""Performs a chat-like interaction with the knowledge abstract.
This generic method retrieves relevant items and generates a response.
Subclasses with complex data structures (like graphs) should override this
to provide better context formatting.
Args:
query: User query string.
top_k: Number of relevant items to retrieve (default: 3).
source_ids: Optional scope forwarded to search() when the type accepts it.
tags: Optional scope forwarded to search() when the type accepts it.
Returns:
An AIMessage object containing the LLM-generated response.
"""
import inspect
# Step 1: Retrieve relevant items from knowledge abstract
search_kwargs: dict[str, Any] = {}
search_params = inspect.signature(type(self).search).parameters
if "source_ids" in search_params:
search_kwargs["source_ids"] = source_ids
if "tags" in search_params:
search_kwargs["tags"] = tags
search_results = self.search(query, top_k, **search_kwargs)
# Step 2: Format context from retrieved items
formatted_context = []
if not search_results:
context = "No relevant information found in the knowledge abstract."
else:
for item in search_results:
if isinstance(item, BaseModel):
formatted_context.append(item.model_dump_json(indent=2))
elif isinstance(item, dict):
formatted_context.append(
json.dumps(item, indent=2, ensure_ascii=False)
)
elif isinstance(item, str):
formatted_context.append(item)
else:
raise ValueError(
"Search results must be Pydantic models, strings, or dicts for formatting."
)
context = "\n---\n".join(formatted_context)
# Step 3: Create QA prompt template and invoke LLM
qa_prompt = ChatPromptTemplate.from_template(
"Based on the following context from the knowledge abstract, answer the user's question.\n\n"
"Context:\n{context}\n\n"
"Question: {question}\n\n"
"Answer:"
)
qa_chain = qa_prompt | self.llm_client
response = qa_chain.invoke({"context": context, "question": query})
# Step 4: Inject retrieved items into response metadata
if not response.additional_kwargs:
response.additional_kwargs = {}
response.additional_kwargs["retrieved_items"] = search_results
# Step 5: Return the AIMessage response
return response
# ==================== Serialization: Orchestrator ====================
def dump(self, folder_path: str | Path) -> None:
"""Saves the entire knowledge abstract (data, metadata, index) to a directory.
This is the main entry point for serialization. It creates a directory structure:
/folder_path
|-- data.json (The structured knowledge data)
|-- metadata.json (Metadata, localized config, timestamps)
|-- /index (Vector store files, e.g., FAISS)
Args:
folder_path: Target directory path.
"""
root = Path(folder_path)
root.mkdir(parents=True, exist_ok=True)
# 1. Save Core Data
self.dump_data(root / "data.json")
# 2. Save Metadata
self.dump_metadata(root / "metadata.json")
# 3. Save Index (Sub-folder)
# We perform a try-catch or check here because building an index is optional
index_path = root / "index"
if not index_path.exists():
index_path.mkdir()
try:
self.dump_index(index_path)
except Exception as e:
# If saving index fails (or isn't implemented/initialized), we treat it as non-fatal
# user can always rebuild_index()
print(f"Warning: Failed to save vector index: {e}")
# 4. Provenance (Optional — graph-family with track_sources)
self._dump_provenance(root)
def load(self, folder_path: str | Path) -> None:
"""Loads the entire knowledge abstract from a directory.
Args:
folder_path: Source directory path.
"""
root = Path(folder_path)
if not root.exists():
raise FileNotFoundError(f"Knowledge abstract directory not found: {root}")
# 1. Load Core Data (Critical)
self.load_data(root / "data.json")
# 2. Load Metadata (Optional but recommended)
meta_path = root / "metadata.json"
if meta_path.exists():
self.load_metadata(meta_path)
# 3. Load Index (Optional)
index_path = root / "index"
if index_path.exists() and any(index_path.iterdir()):
try:
self.load_index(index_path)
except Exception as e:
print(
f"Warning: Failed to load vector index: {e}. You may need to rebuild_index()."
)
# 4. Provenance (Optional — graph-family with track_sources)
self._load_provenance(root)
# ==================== Serialization: Components ====================
def dump_data(self, file_path: str | Path) -> None:
"""Saves the pure knowledge data to disk as JSON.
Args:
file_path: Target file path for saving data (e.g., "data.json").
"""
path = Path(file_path)
path.parent.mkdir(parents=True, exist_ok=True)
# Use Pydantic's model_dump to serialize the schema
export_data = self.data.model_dump()
with open(path, "w", encoding="utf-8") as f:
json.dump(export_data, f, indent=2, ensure_ascii=False, default=str)
def load_data(self, file_path: str | Path) -> None:
"""Loads data from disk and restores internal state.
Args:
file_path: Source file path containing data (e.g., "data.json").
"""
path = Path(file_path)
if not path.exists():
raise FileNotFoundError(f"Data file not found: {path}")
with open(path, "r", encoding="utf-8") as f:
raw_data = json.load(f)
# Validate and Set State
validated_data = self._data_schema.model_validate(raw_data)
self._set_data_state(validated_data)
def dump_metadata(self, file_path: str | Path) -> None:
"""Saves metadata (timestamps, configs) to disk.
Args:
file_path: Target file path.
"""
path = Path(file_path)
with open(path, "w", encoding="utf-8") as f:
json.dump(self.metadata, f, indent=2, ensure_ascii=False, default=str)
def load_metadata(self, file_path: str | Path) -> None:
"""Loads metadata from disk.
Args:
file_path: Source file path.
"""
path = Path(file_path)
with open(path, "r", encoding="utf-8") as f:
params = json.load(f)
# Timestamps are serialized as ISO strings (dump_metadata uses
# default=str). Restore them to datetime so a loaded instance behaves
# like a freshly created one; otherwise comparisons such as the
# min(created_at, ...) merge in __add__ raise TypeError when a loaded
# instance (str) is combined with a fresh one (datetime).
for key in ("created_at", "updated_at"):
value = params.get(key)
if isinstance(value, str):
try:
params[key] = datetime.fromisoformat(value)
except ValueError:
pass
self.metadata.update(params)
self._restore_observation_context()
def _restore_observation_context(self) -> None:
"""Copy persisted observation fields back onto instance attributes."""
if "observation_time" in self.metadata and hasattr(self, "observation_time"):
self.observation_time = self.metadata["observation_time"]
if "observation_location" in self.metadata and hasattr(
self, "observation_location"
):
self.observation_location = self.metadata["observation_location"]
@abstractmethod
def dump_index(self, folder_path: str | Path) -> None:
"""Save vector index to disk.
Subclasses must implement to support their specific vector store
(FAISS, Chroma, Pinecone, etc.).
Args:
folder_path: Target folder path for saving index files.
"""
@abstractmethod
def load_index(self, folder_path: str | Path) -> None:
"""Load vector index from disk.
Subclasses must implement to support their specific vector store
(FAISS, Chroma, Pinecone, etc.).
Args:
folder_path: Source folder path containing index files.
"""
# ==================== Operator Overloads ====================
def __add__(self, other: "BaseAutoType[T]") -> "BaseAutoType[T]":
"""Operator overload for '+' to merge two knowledge instances.
Creates a new knowledge instance by merging the data from both instances.
The new instance inherits configuration from the left operand (self).
Usage:
>>> kb1 = AutoList(PersonSchema, ...)
>>> kb2 = AutoList(PersonSchema, ...)
>>> kb3 = kb1 + kb2 # ✅ Same schema, creates merged instance
>>>
>>> kb4 = AutoList(CompanySchema, ...)
>>> kb5 = kb1 + kb4 # ❌ TypeError: Different schemas
Args:
other: Another knowledge instance of the same type to merge with.
Returns:
A new knowledge instance containing merged data.
Raises:
TypeError: If other is not an instance of the same knowledge class or has different data schema.
"""
# Check 1: Both must be instances of the same knowledge class
if not isinstance(other, self.__class__):
raise TypeError(
f"Cannot add {type(other).__name__} to {type(self).__name__}. "
f"Both operands must be instances of the same knowledge class."
)
# Check 2: Both must have the same data schema
if self._data_schema != other._data_schema:
raise TypeError(
f"Cannot add knowledge instances with different data schemas. "
f"Left schema: {self._data_schema.__name__}, "
f"Right schema: {other._data_schema.__name__}. "
f"Both operands must have the same data schema to be merged."
)
# Merge the data from both instances
merged_data = self.merge_batch_data([self._data, other._data])
# Create a new instance with the same configuration
new_instance = self._create_empty_instance()
# Set the merged data using hook and update metadata
new_instance._set_data_state(merged_data)
new_instance.metadata["created_at"] = min(
self.metadata["created_at"], other.metadata["created_at"]
)
new_instance.metadata["updated_at"] = datetime.now()
return new_instance