Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
feat: implement incremental delta processing
ZohaibHassan1659ff25fc0678
diff --git a/docs/reference/change_management.md b/docs/reference/change_management.mdindex 0c2b24b2..012072f9 100644--- a/docs/reference/change_management.md+++ b/docs/reference/change_management.md@@ -245,6 +245,49 @@ print(f"Axioms modified: {diff['axioms_modified']}")---+## Incremental / Delta processing++For large-scale knowledge graphs, reprocessing the entire dataset on every update is computationally expensive.+Semantica supports **Delta-Aware Pipelines**, allowing you to compute the exact differences (added and removed triples)+between the two graph snapshots and run validation, enrichment, or export jobs *only* on the changes.++### Delta Pipeline Example++```python+from semantica.change_management import TemporalVersionManager+from semantica.pipeline import PipelineBuilder, ExecutionEngine++# a. Initialize your managers+version_manager = TemporalVersionManager(store_graph="kg_version.db")+triplet_store = get_my_triplet_store()++# b. Build a delta-aware pipeline+builder = PipelineBuilder()+builder.add_step(+ step_name="validate_changes",+ step_type="validation",+ handler=my_validation_handler,+ delta_mode=True, # Enables incremental processing+ base_version_id="v1.0",+ target_version_id="v1.1",+)++pipeline = builder.build("incremental_nightly_job")++# c. Execute the pipeline+engine = ExecutionEngine()++# The engine dynamically intercepts the flow, computes the delta on the+# database backend, and passes ONLY the changed triples to the handler.+result = engine.execute_pipeline(+ pipeline,+ data={}, # Is ignored in delta mode+ version_manager=version_manager,+ triplet_store=triplet_store+)+```+---+## Data Integrity### compute_checksumdiff --git a/docs/reference/pipeline.md b/docs/reference/pipeline.mdindex 74eea1dd..ca73cef5 100644--- a/docs/reference/pipeline.md+++ b/docs/reference/pipeline.md@@ -131,7 +131,7 @@ The **Pipeline Module** provides a robust orchestration engine for building, exe### Types- `Pipeline` — Pipeline definition dataclass-- `PipelineStep` — Pipeline step definition dataclass+- `PipelineStep` — Pipeline step definition dataclass, Supports `delta_mode` (bool), `base_version_id` (str), and `target_version_id` (str) for incremental processing.- `StepStatus` — Enum: `pending`, `running`, `completed`, `failed`, `skipped`- `ExecutionResult` — Execution result dataclass- `PipelineStatus` — Enum: `pending`, `running`, `paused`, `completed`, `failed`, `stopped`@@ -405,6 +405,47 @@ result = engine.execute_pipeline(pipeline, data={"path": "document.pdf"})---+### Incremental / Delta-Aware Pipeline++Use `delta_mode` to process only the differences between two graph versions, drastically reducing compute costs for large datasets.++```python+from semantica.pipeline import PipelineBuilder, ExecutionEngine++builder = (+ PipelineBuilder()+ # Adding delta_mode=True tells the execution engine to intercept this step,+ # compute the diff between v1 and v2, and pass ONLY the delta payload to the handler.+ .add_step(+ "validate_diff",+ "validation",+ delta_mode=True,+ base_version_id="v1",+ target_version_id="v2",+ handler=diff_validator+ )+ .add_step(+ "alert_on_removals",+ "alerting",+ dependencies=["validate_diff"],+ handler=alert_handler+ )+)++pipeline = builder.build(name="IncrementalJob")+engine = ExecutionEngine()++# Execution requires version_manager and triplet_store injected via options+# so the engine can resolve URIs and compute the graph differences natively.+result = engine.execute_pipeline(+ pipeline,+ version_manager=my_version_manager,+ triplet_store=my_triplet_store+)+```++---+## Best Practices1. **Idempotency**: Ensure steps are idempotent (can be run multiple times without side effects) to support retries.diff --git a/semantica/change_management/managers.py b/semantica/change_management/managers.pyindex 6941c837..0dc24d04 100644--- a/semantica/change_management/managers.py+++ b/semantica/change_management/managers.py@@ -6,7 +6,7 @@ and ontologies, with comprehensive change tracking, persistent storage, and audiKey Features:- Enhanced TemporalVersionManager for knowledge graphs- - Enhanced VersionManager for ontologies+ - Enhanced VersionManager for ontologies- Detailed diff algorithms for entities and relationships- Structural comparison for ontology elements- Integration with storage backends and metadata@@ -24,7 +24,13 @@ from datetime import datetimefrom typing import Any, Dict, List, Optionalfrom .change_log import ChangeLogEntry-from .version_storage import VersionStorage, InMemoryVersionStorage, SQLiteVersionStorage, compute_checksum, verify_checksum+from .version_storage import (+ VersionStorage,+ InMemoryVersionStorage,+ SQLiteVersionStorage,+ compute_checksum,+ verify_checksum,+)from ..utils.exceptions import ValidationError, ProcessingErrorfrom ..utils.logging import get_logger@@ -32,20 +38,20 @@ from ..utils.logging import get_loggerclass BaseVersionManager(ABC):"""Abstract base class for enhanced version managers.-+Provides common functionality for version management across different data types."""-+def __init__(self, storage_path: Optional[str] = None):"""Initialize base version manager.-+Args:storage_path: Path to SQLite database for persistent storage.If None, uses in-memory storage."""self.logger = get_logger(self.__class__.__name__.lower())-+# Initialize storage backendif storage_path:self.storage = SQLiteVersionStorage(storage_path)@@ -53,25 +59,29 @@ class BaseVersionManager(ABC):else:self.storage = InMemoryVersionStorage()self.logger.info("Initialized with in-memory storage")-+@abstractmethod- def create_snapshot(self, data: Any, version_label: str, author: str, description: str, **options) -> Dict[str, Any]:+ def create_snapshot(+ self, data: Any, version_label: str, author: str, description: str, **options+ ) -> Dict[str, Any]:"""Create a versioned snapshot of the data."""pass-+@abstractmethod- def compare_versions(self, version1: Any, version2: Any, **options) -> Dict[str, Any]:+ def compare_versions(+ self, version1: Any, version2: Any, **options+ ) -> Dict[str, Any]:"""Compare two versions and return detailed differences."""pass-+def list_versions(self) -> List[Dict[str, Any]]:"""List all version snapshots."""return self.storage.list_all()-+def get_version(self, label: str) -> Optional[Dict[str, Any]]:"""Retrieve specific version by label."""return self.storage.get(label)-+def verify_checksum(self, snapshot: Dict[str, Any]) -> bool:"""Verify the integrity of a snapshot using its checksum."""return verify_checksum(snapshot)@@ -80,10 +90,10 @@ class BaseVersionManager(ABC):class TemporalVersionManager(BaseVersionManager):"""Temporal version management engine for knowledge graphs.-+Provides comprehensive version/snapshot management capabilities includingpersistent storage, detailed change tracking, and audit trails.-+Features:- Persistent snapshot storage (SQLite or in-memory)- Detailed change tracking with entity-level diffs@@ -92,11 +102,11 @@ class TemporalVersionManager(BaseVersionManager):- Version comparison with backward compatibility- Input validation and security features"""-+def __init__(self, storage_path: Optional[str] = None, **config):"""Initialize enhanced temporal version manager.-+Args:storage_path: Path to SQLite database file for persistent storage.If None, uses in-memory storage@@ -104,39 +114,37 @@ class TemporalVersionManager(BaseVersionManager):"""super().__init__(storage_path)self.config = config-+def create_snapshot(- self,- graph: Dict[str, Any],- version_label: str,- author: str,+ self,+ graph: Dict[str, Any],+ version_label: str,+ author: str,description: str,- **options+ **options,) -> Dict[str, Any]:"""Create and store snapshot with checksum and metadata.-+Args:graph: Knowledge graph dict with "entities" and "relationships"version_label: Version string (e.g., "v1.0")author: Email address of the change authordescription: Change description (max 500 chars)**options: Additional options-+Returns:dict: Snapshot with metadata and checksum-+Raises:ValidationError: If input validation failsProcessingError: If storage operation fails"""# Validate inputschange_entry = ChangeLogEntry(- timestamp=datetime.now().isoformat(),- author=author,- description=description+ timestamp=datetime.now().isoformat(), author=author, description=description)-+# Create snapshotsnapshot = {"label": version_label,@@ -145,18 +153,18 @@ class TemporalVersionManager(BaseVersionManager):"description": change_entry.description,"entities": graph.get("entities", []).copy(),"relationships": graph.get("relationships", []).copy(),- "metadata": options.get("metadata", {})+ "metadata": options.get("metadata", {}),}-+# Compute and add checksumsnapshot["checksum"] = compute_checksum(snapshot)-+# Store snapshotself.storage.save(snapshot)-+self.logger.info(f"Created snapshot '{version_label}' by {author}")return snapshot-+def compare_versions(self,v1_label_or_dict,@@ -166,13 +174,13 @@ class TemporalVersionManager(BaseVersionManager):) -> Dict[str, Any]:"""Compare two graph versions with detailed entity-level differences.-+Args:v1_label_or_dict: First version (label string or snapshot dict)v2_label_or_dict: Second version (label string or snapshot dict)comparison_metrics: List of metrics to calculate (optional, unused)**options: Additional comparison options (unused)-+Returns:dict: Detailed version comparison results"""@@ -183,17 +191,17 @@ class TemporalVersionManager(BaseVersionManager):raise ValidationError(f"Version not found: {v1_label_or_dict}")else:version1 = v1_label_or_dict-+if isinstance(v2_label_or_dict, str):version2 = self.storage.get(v2_label_or_dict)if not version2:raise ValidationError(f"Version not found: {v2_label_or_dict}")else:version2 = v2_label_or_dict-+# Compute detailed diffdetailed_diff = self._compute_detailed_diff(version1, version2)-+# Maintain backward compatibility with summarysummary = {"entities_added": len(detailed_diff["entities_added"]),@@ -201,132 +209,197 @@ class TemporalVersionManager(BaseVersionManager):"entities_modified": len(detailed_diff["entities_modified"]),"relationships_added": len(detailed_diff["relationships_added"]),"relationships_removed": len(detailed_diff["relationships_removed"]),- "relationships_modified": len(detailed_diff["relationships_modified"])+ "relationships_modified": len(detailed_diff["relationships_modified"]),}-+return {"version1": version1.get("label", "unknown"),"version2": version2.get("label", "unknown"),"summary": summary,- **detailed_diff+ **detailed_diff,}-- def _compute_detailed_diff(self, version1: Dict[str, Any], version2: Dict[str, Any]) -> Dict[str, Any]:++ def _compute_detailed_diff(+ self, version1: Dict[str, Any], version2: Dict[str, Any]+ ) -> Dict[str, Any]:"""Compute detailed entity and relationship differences between versions.-+Args:version1: First version snapshotversion2: Second version snapshot-+Returns:Dict with detailed diff information"""- entities1 = {e.get("id", str(i)): e for i, e in enumerate(version1.get("entities", []))}- entities2 = {e.get("id", str(i)): e for i, e in enumerate(version2.get("entities", []))}-- relationships1 = {self._relationship_key(r): r for r in version1.get("relationships", [])}- relationships2 = {self._relationship_key(r): r for r in version2.get("relationships", [])}-+ entities1 = {+ e.get("id", str(i)): e for i, e in enumerate(version1.get("entities", []))+ }+ entities2 = {+ e.get("id", str(i)): e for i, e in enumerate(version2.get("entities", []))+ }++ relationships1 = {+ self._relationship_key(r): r for r in version1.get("relationships", [])+ }+ relationships2 = {+ self._relationship_key(r): r for r in version2.get("relationships", [])+ }+# Entity differencesentity_ids1 = set(entities1.keys())entity_ids2 = set(entities2.keys())-+entities_added = [entities2[eid] for eid in entity_ids2 - entity_ids1]entities_removed = [entities1[eid] for eid in entity_ids1 - entity_ids2]-+entities_modified = []for eid in entity_ids1 & entity_ids2:if entities1[eid] != entities2[eid]:changes = self._compute_entity_changes(entities1[eid], entities2[eid])- entities_modified.append({- "id": eid,- "before": entities1[eid],- "after": entities2[eid],- "changes": changes- })-+ entities_modified.append(+ {+ "id": eid,+ "before": entities1[eid],+ "after": entities2[eid],+ "changes": changes,+ }+ )+# Relationship differencesrel_keys1 = set(relationships1.keys())rel_keys2 = set(relationships2.keys())-+relationships_added = [relationships2[key] for key in rel_keys2 - rel_keys1]relationships_removed = [relationships1[key] for key in rel_keys1 - rel_keys2]-+relationships_modified = []for key in rel_keys1 & rel_keys2:if relationships1[key] != relationships2[key]:- changes = self._compute_relationship_changes(relationships1[key], relationships2[key])- relationships_modified.append({- "key": key,- "before": relationships1[key],- "after": relationships2[key],- "changes": changes- })-+ changes = self._compute_relationship_changes(+ relationships1[key], relationships2[key]+ )+ relationships_modified.append(+ {+ "key": key,+ "before": relationships1[key],+ "after": relationships2[key],+ "changes": changes,+ }+ )+return {"entities_added": entities_added,"entities_removed": entities_removed,"entities_modified": entities_modified,"relationships_added": relationships_added,"relationships_removed": relationships_removed,- "relationships_modified": relationships_modified+ "relationships_modified": relationships_modified,}-+def _relationship_key(self, relationship: Dict[str, Any]) -> str:"""Generate a unique key for a relationship."""source = relationship.get("source", "")target = relationship.get("target", "")rel_type = relationship.get("type", relationship.get("relationship", ""))return f"{source}|{rel_type}|{target}"-- def _compute_entity_changes(self, entity1: Dict[str, Any], entity2: Dict[str, Any]) -> Dict[str, Any]:++ def _compute_entity_changes(+ self, entity1: Dict[str, Any], entity2: Dict[str, Any]+ ) -> Dict[str, Any]:"""Compute changes between two entity versions."""changes = {}all_keys = set(entity1.keys()) | set(entity2.keys())-+for key in all_keys:val1 = entity1.get(key)val2 = entity2.get(key)-+if val1 != val2:changes[key] = {"from": val1, "to": val2}-+return changes-- def _compute_relationship_changes(self, rel1: Dict[str, Any], rel2: Dict[str, Any]) -> Dict[str, Any]:++ def _compute_relationship_changes(+ self, rel1: Dict[str, Any], rel2: Dict[str, Any]+ ) -> Dict[str, Any]:"""Compute changes between two relationship versions."""changes = {}all_keys = set(rel1.keys()) | set(rel2.keys())-+for key in all_keys:val1 = rel1.get(key)val2 = rel2.get(key)-+if val1 != val2:changes[key] = {"from": val1, "to": val2}-+return changes++ def prune_versions(self, keep_last_n: int = 5, triplet_store: Any = None) -> Dict [str, Any]:+ """+ Prune old snapshots, keeping only the most recent N versions.+ Optionally deletes the backend graphs from the triplet store to free space.++ Args:+ keep_last_n: Number of recent versions to retain.+ triplet_store: Optional TripletScore instance to execute DROP GRAPH.++ Returns:+ Dict containing counts and labels of pruned versions.+ """++ all_versions = self.list_versions()+ all_versions.sort(key = lambda x: x.get("timestamp", ""), reverse=True)++ versions_to_delete = all_versions[keep_last_n:]+ deleted_labels = []++ for v in versions_to_delete:+ label = v.get("label")+ graph_uri = v.get("graph_uri")++ # delete metadata from SQLite / In-Memory+ if self.storage.delete(label):+ deleted_labels.append(label)++ # Clean up the actual graph if provided+ if triplet_store and graph_uri:+ try:+ triplet_store.execute_query(f"DROP SILENT GRAPH {graph_uri}")+ self.logger.info(f"Dropped obsolete graph {graph_uri} from store")+ except Exception as e:+ self.logger.warning(f"Failed to drop graph {graph_uri} during pruning: {e}")++ self.logger.info(f"Pruned {len(deleted_labels)} old versions, kept {keep_last_n}")+ return {+ "pruned_count": len(deleted_labels),+ "pruned_versions": deleted_labels,+ "retained_count": len(all_versions) - len(deleted_labels)+ }+++class OntologyVersionManager(BaseVersionManager):"""Version management for ontologies with structural comparison.-+Provides comprehensive version management for ontologies includingdetailed structural analysis and change tracking.-+Features:- Structural comparison of ontology elements- Detailed diff for classes, properties, individuals, axioms- Persistent storage with metadata- Change tracking and audit trails"""-+def __init__(self, storage_path: Optional[str] = None, **config):"""Initialize enhanced version manager.-+Args:storage_path: Path to SQLite database file for persistent storage.If None, uses in-memory storage@@ -335,35 +408,33 @@ class OntologyVersionManager(BaseVersionManager):super().__init__(storage_path)self.config = configself.versions = {} # In-memory version tracking for compatibility-+def create_snapshot(self,ontology_data: Dict[str, Any],version_label: str,author: str,description: str,- **options+ **options,) -> Dict[str, Any]:"""Create ontology version snapshot.-+Args:ontology_data: Ontology data dictionaryversion_label: Version string (e.g., "v1.0")author: Email address of the change authordescription: Change description**options: Additional options including metadata-+Returns:dict: Ontology version snapshot"""# Validate inputschange_entry = ChangeLogEntry(- timestamp=datetime.now().isoformat(),- author=author,- description=description+ timestamp=datetime.now().isoformat(), author=author, description=description)-+# Create snapshotsnapshot = {"label": version_label,@@ -373,108 +444,112 @@ class OntologyVersionManager(BaseVersionManager):"ontology_iri": ontology_data.get("uri", ""),"version_info": ontology_data.get("version_info", {}),"structure": ontology_data.get("structure", {}),- "metadata": options.get("metadata", {})+ "metadata": options.get("metadata", {}),}-+# Compute and add checksumsnapshot["checksum"] = compute_checksum(snapshot)-+# Store snapshotself.storage.save(snapshot)-+# Also store in memory for compatibilityself.versions[version_label] = snapshot-+self.logger.info(f"Created ontology snapshot '{version_label}' by {author}")return snapshot-- def compare_versions(self, version1: str, version2: str, **options) -> Dict[str, Any]:++ def compare_versions(+ self, version1: str, version2: str, **options+ ) -> Dict[str, Any]:"""Compare two ontology versions with detailed structural analysis.-+Args:version1: First version labelversion2: Second version label**options: Additional comparison options-+Returns:Detailed comparison results including structural differences"""# Get versions from storagev1_snapshot = self.storage.get(version1)v2_snapshot = self.storage.get(version2)-+if not v1_snapshot:raise ValidationError(f"Version not found: {version1}")if not v2_snapshot:raise ValidationError(f"Version not found: {version2}")-+# Basic metadata comparisonmetadata_changes = {}if v1_snapshot.get("ontology_iri") != v2_snapshot.get("ontology_iri"):metadata_changes["ontology_iri"] = {"from": v1_snapshot.get("ontology_iri"),- "to": v2_snapshot.get("ontology_iri")+ "to": v2_snapshot.get("ontology_iri"),}if v1_snapshot.get("version_info") != v2_snapshot.get("version_info"):metadata_changes["version_info"] = {"from": v1_snapshot.get("version_info"),- "to": v2_snapshot.get("version_info")+ "to": v2_snapshot.get("version_info"),}-+# Structural comparisonstructural_diff = self._compare_ontology_structures(v1_snapshot, v2_snapshot)-+return {"version1": version1,"version2": version2,"metadata_changes": metadata_changes,- **structural_diff+ **structural_diff,}-- def _compare_ontology_structures(self, v1_snapshot: Dict[str, Any], v2_snapshot: Dict[str, Any]) -> Dict[str, Any]:++ def _compare_ontology_structures(+ self, v1_snapshot: Dict[str, Any], v2_snapshot: Dict[str, Any]+ ) -> Dict[str, Any]:"""Compare structural elements between two ontology versions.-+Args:v1_snapshot: First ontology version snapshotv2_snapshot: Second ontology version snapshot-+Returns:Dictionary with structural differences"""# Extract structural informationv1_structure = v1_snapshot.get("structure", {})v2_structure = v2_snapshot.get("structure", {})-+# Compare classesv1_classes = set(v1_structure.get("classes", []))v2_classes = set(v2_structure.get("classes", []))-+classes_added = list(v2_classes - v1_classes)classes_removed = list(v1_classes - v2_classes)-+# Compare propertiesv1_properties = set(v1_structure.get("properties", []))v2_properties = set(v2_structure.get("properties", []))-+properties_added = list(v2_properties - v1_properties)properties_removed = list(v1_properties - v2_properties)-+# Compare individualsv1_individuals = set(v1_structure.get("individuals", []))v2_individuals = set(v2_structure.get("individuals", []))-+individuals_added = list(v2_individuals - v1_individuals)individuals_removed = list(v1_individuals - v2_individuals)-+# Compare axioms/rulesv1_axioms = set(v1_structure.get("axioms", []))v2_axioms = set(v2_structure.get("axioms", []))-+axioms_added = list(v2_axioms - v1_axioms)axioms_removed = list(v1_axioms - v2_axioms)-+return {"classes_added": classes_added,"classes_removed": classes_removed,@@ -492,6 +567,6 @@ class OntologyVersionManager(BaseVersionManager):"individuals_added": len(individuals_added),"individuals_removed": len(individuals_removed),"axioms_added": len(axioms_added),- "axioms_removed": len(axioms_removed)- }+ "axioms_removed": len(axioms_removed),+ },}diff --git a/semantica/change_management/version_storage.py b/semantica/change_management/version_storage.pyindex aa2ee87b..b5e7e4b4 100644--- a/semantica/change_management/version_storage.py+++ b/semantica/change_management/version_storage.py@@ -39,72 +39,105 @@ from ..utils.exceptions import ProcessingError, ValidationErrorfrom ..utils.logging import get_logger++def create_graph_snapshot_record(+ version_id: str,+ graph_uri: str,+ author: str = "system",+ description: str = "Graph snapshot",+ metadata: Optional[Dict[str, Any]] = None+) -> Dict[str, Any]:+ """+ Creates a standardized snapshot metadata record for a named graph.++ Args:+ version_id: Unique identifier for this snapshot+ graph_uri: The underlying named graph URI in the triplet store+ author: Creator of the snapshot+ description: Purpose or context of the snapshot+ metadata: Additional tags or piepline context+ """++ record = {+ "label": version_id,+ "version_id": version_id,+ "graph_uri": graph_uri,+ "timestamp": datetime.now().isoformat(),+ "author": author,+ "description": description,+ "metadata": metadata or {},+ }++ record["checksum"] = compute_checksum(record)+ return record++class VersionStorage(ABC):"""Abstract base class for version storage backends.-+This interface defines the contract that all storage implementationsmust follow for version management operations."""-+@abstractmethoddef save(self, snapshot: Dict[str, Any]) -> None:"""Save a version snapshot.-+Args:snapshot: Version snapshot dictionary with metadata-+Raises:ValidationError: If snapshot data is invalidProcessingError: If save operation fails"""pass-+@abstractmethoddef get(self, label: str) -> Optional[Dict[str, Any]]:"""Retrieve a version snapshot by label.-+Args:label: Version label to retrieve-+Returns:Snapshot dictionary or None if not found"""pass-+@abstractmethoddef list_all(self) -> List[Dict[str, Any]]:"""List all version snapshots.-+Returns:List of snapshot metadata dictionaries"""pass-+@abstractmethoddef exists(self, label: str) -> bool:"""Check if a version exists.-+Args:label: Version label to check-+Returns:True if version exists, False otherwise"""pass-+@abstractmethoddef delete(self, label: str) -> bool:"""Delete a version snapshot.-+Args:label: Version label to delete-+Returns:True if deleted, False if not found"""@@ -114,31 +147,31 @@ class VersionStorage(ABC):class InMemoryVersionStorage(VersionStorage):"""In-memory version storage implementation.-+This implementation stores all version data in memory using a dictionary.Data is lost when the process ends."""-+def __init__(self):"""Initialize in-memory storage."""self._storage: Dict[str, Dict[str, Any]] = {}self._lock = threading.RLock()self.logger = get_logger("in_memory_storage")-+def save(self, snapshot: Dict[str, Any]) -> None:"""Save snapshot to memory."""label = snapshot.get("label")if not label:raise ValidationError("Snapshot must have a 'label' field")-+with self._lock:if label in self._storage:raise ValidationError(f"Version '{label}' already exists")-+# Deep copy to prevent external modificationsself._storage[label] = json.loads(json.dumps(snapshot))self.logger.debug(f"Saved version '{label}' to memory")-+def get(self, label: str) -> Optional[Dict[str, Any]]:"""Retrieve snapshot from memory."""with self._lock:@@ -147,7 +180,7 @@ class InMemoryVersionStorage(VersionStorage):# Return deep copy to prevent external modificationsreturn json.loads(json.dumps(snapshot))return None-+def list_all(self) -> List[Dict[str, Any]]:"""List all snapshots in memory."""with self._lock:@@ -156,21 +189,23 @@ class InMemoryVersionStorage(VersionStorage):for label, snapshot in self._storage.items():metadata = {"label": snapshot.get("label"),+ "version_id": snapshot.get("version_id", snapshot.get("label")),+ "graph_uri": snapshot.get("graph_uri"),"timestamp": snapshot.get("timestamp"),"author": snapshot.get("author"),"description": snapshot.get("description"),"checksum": snapshot.get("checksum"),"entity_count": len(snapshot.get("entities", [])),- "relationship_count": len(snapshot.get("relationships", []))- }+ "relationship_count": len(snapshot.get("relationships", [])),+ }metadata_list.append(metadata)return metadata_list-+def exists(self, label: str) -> bool:"""Check if version exists in memory."""with self._lock:return label in self._storage-+def delete(self, label: str) -> bool:"""Delete version from memory."""with self._lock:@@ -184,28 +219,28 @@ class InMemoryVersionStorage(VersionStorage):class SQLiteVersionStorage(VersionStorage):"""SQLite-based persistent version storage implementation.-+This implementation stores version data in a SQLite database file,providing persistence across process restarts."""-+def __init__(self, storage_path: str):"""Initialize SQLite storage.-+Args:storage_path: Path to SQLite database file"""self.storage_path = Path(storage_path)self._lock = threading.RLock()self.logger = get_logger("sqlite_storage")-+# Create directory if it doesn't existself.storage_path.parent.mkdir(parents=True, exist_ok=True)-+# Initialize databaseself._init_database()-+def _init_database(self) -> None:"""Initialize SQLite database schema."""with self._lock:@@ -227,67 +262,73 @@ class SQLiteVersionStorage(VersionStorage):self.logger.debug(f"Initialized SQLite database at {self.storage_path}")finally:conn.close()-+def save(self, snapshot: Dict[str, Any]) -> None:"""Save snapshot to SQLite database."""label = snapshot.get("label")if not label:raise ValidationError("Snapshot must have a 'label' field")-+with self._lock:conn = sqlite3.connect(str(self.storage_path))try:cursor = conn.cursor()-+# Check if version already existscursor.execute("SELECT label FROM versions WHERE label = ?", (label,))if cursor.fetchone():raise ValidationError(f"Version '{label}' already exists")-+# Insert new version- cursor.execute("""+ cursor.execute(+ """INSERT INTO versions(label, timestamp, author, description, checksum, snapshot_data, created_at)VALUES (?, ?, ?, ?, ?, ?, ?)- """, (- label,- snapshot.get("timestamp", ""),- snapshot.get("author", ""),- snapshot.get("description", ""),- snapshot.get("checksum", ""),- json.dumps(snapshot),- datetime.now().isoformat()- ))-+ """,+ (+ label,+ snapshot.get("timestamp", ""),+ snapshot.get("author", ""),+ snapshot.get("description", ""),+ snapshot.get("checksum", ""),+ json.dumps(snapshot),+ datetime.now().isoformat(),+ ),+ )+conn.commit()self.logger.debug(f"Saved version '{label}' to SQLite database")-+except sqlite3.Error as e:raise ProcessingError(f"Failed to save version to database: {e}")finally:conn.close()-+def get(self, label: str) -> Optional[Dict[str, Any]]:"""Retrieve snapshot from SQLite database."""with self._lock:conn = sqlite3.connect(str(self.storage_path))try:cursor = conn.cursor()- cursor.execute("""+ cursor.execute(+ """SELECT snapshot_data FROM versions WHERE label = ?- """, (label,))-+ """,+ (label,),+ )+row = cursor.fetchone()if not row:return None-+return json.loads(row[0])-+except sqlite3.Error as e:raise ProcessingError(f"Failed to retrieve version from database: {e}")finally:conn.close()-+def list_all(self) -> List[Dict[str, Any]]:"""List all snapshots in SQLite database."""with self._lock:@@ -297,29 +338,31 @@ class SQLiteVersionStorage(VersionStorage):cursor.execute("""SELECT snapshot_data FROM versions ORDER BY timestamp DESC""")-+metadata_list = []for row in cursor.fetchall():snapshot = json.loads(row[0])-+metadata = {"label": snapshot.get("label"),+ "version_id": snapshot.get("version_id", snapshot.get("label")),+ "graph_uri": snapshot.get("graph_uri"),"timestamp": snapshot.get("timestamp"),"author": snapshot.get("author"),"description": snapshot.get("description"),"checksum": snapshot.get("checksum"),"entity_count": len(snapshot.get("entities", [])),- "relationship_count": len(snapshot.get("relationships", []))+ "relationship_count": len(snapshot.get("relationships", [])),}metadata_list.append(metadata)-+return metadata_list-+except sqlite3.Error as e:raise ProcessingError(f"Failed to list versions from database: {e}")finally:conn.close()-+def exists(self, label: str) -> bool:"""Check if version exists in SQLite database."""with self._lock:@@ -332,7 +375,7 @@ class SQLiteVersionStorage(VersionStorage):raise ProcessingError(f"Failed to check version existence: {e}")finally:conn.close()-+def delete(self, label: str) -> bool:"""Delete version from SQLite database."""with self._lock:@@ -342,12 +385,12 @@ class SQLiteVersionStorage(VersionStorage):cursor.execute("DELETE FROM versions WHERE label = ?", (label,))deleted = cursor.rowcount > 0conn.commit()-+if deleted:self.logger.debug(f"Deleted version '{label}' from SQLite database")-+return deleted-+except sqlite3.Error as e:raise ProcessingError(f"Failed to delete version from database: {e}")finally:@@ -357,35 +400,35 @@ class SQLiteVersionStorage(VersionStorage):def compute_checksum(data: Dict[str, Any]) -> str:"""Compute SHA-256 checksum for version data.-+Args:data: Dictionary containing version data-+Returns:SHA-256 checksum as hexadecimal string"""# Create a deterministic JSON representation- json_str = json.dumps(data, sort_keys=True, separators=(',', ':'))- return hashlib.sha256(json_str.encode('utf-8')).hexdigest()+ json_str = json.dumps(data, sort_keys=True, separators=(",", ":"))+ return hashlib.sha256(json_str.encode("utf-8")).hexdigest()def verify_checksum(snapshot: Dict[str, Any]) -> bool:"""Verify the integrity of a snapshot using its checksum.-+Args:snapshot: Snapshot dictionary with checksum field-+Returns:True if checksum is valid, False otherwise"""stored_checksum = snapshot.get("checksum")if not stored_checksum:return False-+# Create copy without checksum for verificationdata_copy = snapshot.copy()data_copy.pop("checksum", None)-+computed_checksum = compute_checksum(data_copy)return stored_checksum == computed_checksumdiff --git a/semantica/pipeline/execution_engine.py b/semantica/pipeline/execution_engine.pyindex 2cef1a37..16485b00 100644--- a/semantica/pipeline/execution_engine.py+++ b/semantica/pipeline/execution_engine.py@@ -141,26 +141,37 @@ class ExecutionEngine:# Register pipeline modules for progress trackingmodule_list = []module_order = {}- if hasattr(pipeline, 'steps') and pipeline.steps:+ if hasattr(pipeline, "steps") and pipeline.steps:for idx, step in enumerate(pipeline.steps):# Extract module name from step- module_name = getattr(step, 'module', None) or getattr(step, 'name', None) or str(step)+ module_name = (+ getattr(step, "module", None)+ or getattr(step, "name", None)+ or str(step)+ )if module_name and module_name not in module_list:module_list.append(module_name)module_order[module_name] = idx-+# If no steps found, try to infer from pipeline structureif not module_list:# Common pipeline modules- module_list = ["ingest", "parse", "normalize", "semantic_extract", "kg", "embeddings"]+ module_list = [+ "ingest",+ "parse",+ "normalize",+ "semantic_extract",+ "kg",+ "embeddings",+ ]module_order = {module: idx for idx, module in enumerate(module_list)}-+# Register pipeline modulesif module_list:self.progress_tracker.register_pipeline_modules(pipeline_id=pipeline_id,module_list=module_list,- module_order=module_order+ module_order=module_order,)# Set status@@ -200,7 +211,7 @@ class ExecutionEngine:status="completed" if metrics["steps_failed"] == 0 else "failed",message=f"Executed {metrics['steps_executed']} steps in {execution_time:.2f}s",)-+# Clear pipeline context when pipeline completesself.progress_tracker.clear_pipeline_context(pipeline_id)@@ -300,11 +311,58 @@ class ExecutionEngine:return current_datadef _execute_step(self, step: PipelineStep, data: Any, **options) -> Any:- """Execute a single step."""+ """+ Execute a single step.++ If delta_mode is enabled for the step, this intercepts the execution to compute+ the delta between the base and target versions, passing only the changes+ (added/removed triples) to the handler.+ """++ if getattr(step, "delta_mode", False):+ self.logger.info(f"Executing step '{step.name}' in incremental delta mode.")++ version_manager = options.get("version_manager") or self.config.get("version_manager")+ triplet_store = options.get("triplet_store") or self.config.get("triplet_store")++ if not version_manager or not triplet_store:+ raise ProcessingError(+ f"Step '{step.name}' requires 'version_manager' and 'triplet_store' "+ f"in execution options for delta processing."+ )++ if not step.base_version_id or not step.target_version_id:+ raise ValidationError(+ f"Step '{step.name}' in delta_mode requires 'base_version_id' "+ f"and 'target_version_id' to be set."+ )+++ base_snap = version_manager.get_version(step.base_version_id)+ target_snap = version_manager.get_version(step.target_version_id)++ if not base_snap:+ raise ValidationError(f"Base version '{step.base_version_id}' not found in storage.")+ if not target_snap:+ raise ValidationError(f"Target version '{step.target_version_id}' not found in storage.")++ base_uri = base_snap.get("graph_uri")+ target_uri = target_snap.get("graph_uri")++ if not base_uri or not target_uri:+ raise ValidationError(+ "Both base and target snapshots must contain a 'graph_uri' "+ "to compute native store deltas."+ )++ self.logger.debug(f"Computing delta between {base_uri} and {target_uri}")+ delta_result = triplet_store.compute_delta(base_uri, target_uri, **options)++ data = delta_result+if step.handler:return step.handler(data, **step.config, **options)else:- # Default: pass data throughreturn datadef _topological_sort(self, steps: List[PipelineStep]) -> List[PipelineStep]:@@ -406,9 +464,9 @@ class ExecutionEngine:return {"total_steps": total_steps,"completed_steps": completed_steps,- "progress_percentage": (completed_steps / total_steps * 100)- if total_steps > 0- else 0.0,+ "progress_percentage": (+ (completed_steps / total_steps * 100) if total_steps > 0 else 0.0+ ),"status": self.pipeline_status.get(pipeline_id, PipelineStatus.PENDING).value,diff --git a/semantica/pipeline/pipeline_builder.py b/semantica/pipeline/pipeline_builder.pyindex 22a2e21c..e4082e23 100644--- a/semantica/pipeline/pipeline_builder.py+++ b/semantica/pipeline/pipeline_builder.py@@ -64,6 +64,9 @@ class PipelineStep:status: StepStatus = StepStatus.PENDINGresult: Any = Noneerror: Optional[Exception] = None+ delta_mode: bool = False+ base_version_id: Optional[str] = None+ target_version_id: Optional[str] = None@dataclass@@ -125,16 +128,23 @@ class PipelineBuilder:Returns:Self for method chaining"""+ delta_mode = config.pop("delta_mode", False)+ base_version_id = config.pop("base_version_id", None)+ target_version_id = config.pop("target_version_id", None)+step = PipelineStep(name=step_name,step_type=step_type,config=config,dependencies=config.get("dependencies", []),handler=config.get("handler"),+ delta_mode = delta_mode,+ base_version_id=base_version_id,+ target_version_id=target_version_id,)self.steps.append(step)- self.logger.debug(f"Added step: {step_name} ({step_type})")+ self.logger.debug(f"Added step: {step_name} ({step_type}) | Delta Mode: {delta_mode}")return self@@ -397,6 +407,9 @@ class PipelineSerializer:"type": step.step_type,"config": step.config,"dependencies": step.dependencies,+ "delta_mode": getattr(step, "delta_mode", False),+ "base_version_id": getattr(step, "base_version_id", None),+ "target_version_id": getattr(step, "target_version_id", None),}for step in pipeline.steps],diff --git a/semantica/triplet_store/triplet_store.py b/semantica/triplet_store/triplet_store.pyindex b689efdf..6ac1fc51 100644--- a/semantica/triplet_store/triplet_store.py+++ b/semantica/triplet_store/triplet_store.py@@ -26,7 +26,7 @@ Author: Semantica ContributorsLicense: MIT"""-from typing import Any, Dict, List, Optional, Union+from typing import Any, Dict, List, Optional, Union, Tripletsfrom ..semantic_extract.triplet_extractor import Tripletfrom ..utils.exceptions import ProcessingError, ValidationError@@ -91,47 +91,44 @@ class TripletStore:try:if self.backend_type == "blazegraph":from .blazegraph_store import BlazegraphStore-+# Merge config with defaultsbackend_config = self.config.copy()if self.endpoint:backend_config["endpoint"] = self.endpointelse:backend_config["endpoint"] = triplet_store_config.get(- "blazegraph_endpoint",- "http://localhost:9999/blazegraph"+ "blazegraph_endpoint", "http://localhost:9999/blazegraph")-+self._store_backend = BlazegraphStore(**backend_config)elif self.backend_type == "jena":from .jena_store import JenaStore-+backend_config = self.config.copy()if self.endpoint:backend_config["endpoint"] = self.endpointelse:backend_config["endpoint"] = triplet_store_config.get(- "jena_endpoint",- "http://localhost:3030/ds"+ "jena_endpoint", "http://localhost:3030/ds")-+self._store_backend = JenaStore(**backend_config)elif self.backend_type == "rdf4j":from .rdf4j_store import RDF4JStore-+backend_config = self.config.copy()if self.endpoint:backend_config["endpoint"] = self.endpointelse:backend_config["endpoint"] = triplet_store_config.get(- "rdf4j_endpoint",- "http://localhost:8080/rdf4j-server"+ "rdf4j_endpoint", "http://localhost:8080/rdf4j-server")-+self._store_backend = RDF4JStore(**backend_config)-+self.logger.info(f"Initialized {self.backend_type} backend")except Exception as e:@@ -139,19 +136,19 @@ class TripletStore:raise ProcessingError(f"Failed to initialize backend: {e}")def store(- self,- knowledge_graph: Union[Dict[str, Any], Any],- ontology: Union[Dict[str, Any], Any],- **options+ self,+ knowledge_graph: Union[Dict[str, Any], Any],+ ontology: Union[Dict[str, Any], Any],+ **options,) -> Dict[str, Any]:"""Store knowledge graph and ontology in the triplet store.-+Args:knowledge_graph: Knowledge graph dictionary or objectontology: Ontology dictionary or object**options: Additional options-+Returns:Operation status"""@@ -160,9 +157,9 @@ class TripletStore:knowledge_graph = knowledge_graph.to_dict()if hasattr(ontology, "to_dict"):ontology = ontology.to_dict()-+triplets = []-+# Standard NamespacesRDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"RDFS_SUBCLASS = "http://www.w3.org/2000/01/rdf-schema#subClassOf"@@ -171,23 +168,23 @@ class TripletStore:OWL_DATATYPE_PROPERTY = "http://www.w3.org/2002/07/owl#DatatypeProperty"RDFS_DOMAIN = "http://www.w3.org/2000/01/rdf-schema#domain"RDFS_RANGE = "http://www.w3.org/2000/01/rdf-schema#range"-+# 1. Process Ontologyclasses = ontology.get("classes", [])properties = ontology.get("properties", [])-+for cls in classes:# Class definitioncls_uri = cls.get("uri") or cls.get("id") or cls.get("name")if not cls_uri:continue-+if not cls_uri.startswith("http") and not cls_uri.startswith("urn:"):- # Fallback if no URI provided- cls_uri = f"urn:class:{cls_uri}"-+ # Fallback if no URI provided+ cls_uri = f"urn:class:{cls_uri}"+triplets.append(Triplet(cls_uri, RDF_TYPE, OWL_CLASS))-+# Hierarchyparent = cls.get("parent") or cls.get("subClassOf")if parent:@@ -195,15 +192,15 @@ class TripletStore:if not parent.startswith("http") and not parent.startswith("urn:"):parent_uri = f"urn:class:{parent}"triplets.append(Triplet(cls_uri, RDFS_SUBCLASS, parent_uri))-+for prop in properties:prop_uri = prop.get("uri") or prop.get("id") or prop.get("name")if not prop_uri:continue-+if not prop_uri.startswith("http") and not prop_uri.startswith("urn:"):- prop_uri = f"urn:property:{prop_uri}"-+ prop_uri = f"urn:property:{prop_uri}"+# Determine property type (Object or Datatype)# Default to ObjectProperty if not specifiedprop_type = prop.get("type", OWL_OBJECT_PROPERTY)@@ -211,25 +208,25 @@ class TripletStore:prop_type = OWL_DATATYPE_PROPERTYelif prop_type == "object":prop_type = OWL_OBJECT_PROPERTY-+triplets.append(Triplet(prop_uri, RDF_TYPE, prop_type))-+if "domain" in prop:domains = prop["domain"]if isinstance(domains, str):domains = [domains]-+for domain in domains:domain_uri = domainif not domain.startswith("http") and not domain.startswith("urn:"):domain_uri = f"urn:class:{domain}"triplets.append(Triplet(prop_uri, RDFS_DOMAIN, domain_uri))-+if "range" in prop:ranges = prop["range"]if isinstance(ranges, str):ranges = [ranges]-+for range_ in ranges:range_uri = range_if not range_.startswith("http") and not range_.startswith("urn:"):@@ -239,28 +236,30 @@ class TripletStore:# 2. Process Knowledge Graphentities = knowledge_graph.get("entities", [])relationships = knowledge_graph.get("relationships", [])-- entity_map = {} # Map IDs to URIs-++ entity_map = {} # Map IDs to URIs+for entity in entities:entity_id = entity.get("id")if not entity_id:continue-+entity_uri = entity.get("uri")if not entity_uri:entity_uri = f"urn:entity:{entity_id}"-+entity_map[entity_id] = entity_uri-+# Entity Typeentity_type = entity.get("type")if entity_type:type_uri = entity_type- if not entity_type.startswith("http") and not entity_type.startswith("urn:"):+ if not entity_type.startswith("http") and not entity_type.startswith(+ "urn:"+ ):type_uri = f"urn:class:{entity_type}"triplets.append(Triplet(entity_uri, RDF_TYPE, type_uri))-+# Entity Propertiesprops = entity.get("properties", {})for k, v in props.items():@@ -268,23 +267,23 @@ class TripletStore:if not k.startswith("http") and not k.startswith("urn:"):prop_uri = f"urn:property:{k}"triplets.append(Triplet(entity_uri, prop_uri, str(v)))-+for rel in relationships:source_id = rel.get("source")target_id = rel.get("target")rel_type = rel.get("type") or rel.get("label")-+if not source_id or not target_id or not rel_type:continue-+source_uri = entity_map.get(source_id, f"urn:entity:{source_id}")target_uri = entity_map.get(target_id, f"urn:entity:{target_id}")rel_uri = rel_typeif not rel_type.startswith("http") and not rel_type.startswith("urn:"):rel_uri = f"urn:property:{rel_type}"-+triplets.append(Triplet(source_uri, rel_uri, target_uri))-+# Bulk load all tripletsreturn self.add_triplets(triplets, **options)@@ -305,10 +304,7 @@ class TripletStore:return self._store_backend.add_triplet(triplet, **options)def add_triplets(- self,- triplets: List[Triplet],- batch_size: int = 1000,- **options+ self, triplets: List[Triplet], batch_size: int = 1000, **options) -> Dict[str, Any]:"""Add multiple triplets to the store (bulk load).@@ -330,10 +326,7 @@ class TripletStore:# Use bulk loader for efficient processingprogress = self.bulk_loader.load_triplets(- valid_triplets,- self._store_backend,- batch_size=batch_size,- **options+ valid_triplets, self._store_backend, batch_size=batch_size, **options)return {@@ -341,7 +334,7 @@ class TripletStore:"total": progress.total_triplets,"processed": progress.loaded_triplets,"failed": progress.failed_triplets,- "batches": progress.total_batches+ "batches": progress.total_batches,}def get_triplets(@@ -364,10 +357,7 @@ class TripletStore:List of matching Triplet objects"""return self._store_backend.get_triplets(- subject=subject,- predicate=predicate,- object=object,- **options+ subject=subject, predicate=predicate, object=object, **options)def delete_triplet(self, triplet: Triplet, **options) -> Dict[str, Any]:@@ -384,10 +374,7 @@ class TripletStore:return self._store_backend.delete_triplet(triplet, **options)def update_triplet(- self,- old_triplet: Triplet,- new_triplet: Triplet,- **options+ self, old_triplet: Triplet, new_triplet: Triplet, **options) -> Dict[str, Any]:"""Update a triplet (atomic delete + add).@@ -406,10 +393,7 @@ class TripletStore:return self.add_triplet(new_triplet, **options)def execute_query(- self,- query: str,- parameters: Optional[Dict[str, Any]] = None,- **options+ self, query: str, parameters: Optional[Dict[str, Any]] = None, **options) -> Any:"""Execute a SPARQL query.@@ -428,12 +412,14 @@ class TripletStore:"""Validate triplet structure."""if not triplet.subject or not triplet.predicate or not triplet.object:return False-+# Check confidence score if present- if hasattr(triplet, 'confidence'):- if triplet.confidence is not None and (triplet.confidence < 0 or triplet.confidence > 1):+ if hasattr(triplet, "confidence"):+ if triplet.confidence is not None and (+ triplet.confidence < 0 or triplet.confidence > 1+ ):return False-+return Truedef get_stats(self) -> Dict[str, Any]:@@ -441,3 +427,84 @@ class TripletStore:if hasattr(self._store_backend, "get_stats"):return self._store_backend.get_stats()return {}++ def compute_delta(+ self, old_graph_uri: str, new_graph_uri: str, **options+ ) -> Dict[str, Any]:+ """+ Compute the delta (added and removed triples) between two graph snapshots.++ Args:+ old_graph_uri: URI of the baseline graph snapshot.+ new_graph_uri: URI of the target graph snapshot+ **options: Additional query execution options++ Returns:+ Dictionary containing added_triples, removed_triples, and counts.+ """++ tracking_id = self.progress_tracker.start_tracking(+ module="triplet_store",+ submodule="COmputeDelta",+ message=f"Computing delta: {old_graph_uri} -> {new_graph_uri}",+ )++ # SPARQL: Triples in the new graph that do not exist in the old graph+ added_query = f"""+ SELECT ?s ?p ?o WHERE {{+ GRAPH <{new_graph_uri} > {{ ?s ?p ?o }}+ FILTER NOT EXISTS {{ GRAPH <{old_graph_uri}> {{ ?s ?o ?p}} }}+ }}+ """++ # // : Triples in the old graph that do not exist in the new graph+ removed_query = f"""+ SELECT ?s ?p ?o WHERE {{+ GRAPH <{old_graph_uri}> {{ ?s ?p ?o }}+ FILTER NOT EXISTS {{ GRAPH <{new_graph_uri}> {{ ?s ?p ?o }} }}+ }}+ """++ try:+ self.progress_tracker.update_tracking(tracking_id, message="Executing added triples query...")+ added_res = self.execute_query(added_query, **options)++ self.progress_tracker.update_tracking(tracking_id, message="Executing removed triples query...")+ removed_res = self.execute_query(removed_query, **options)++ def extract_triplets(bindings):+ triplets = []+ for b in bindings:+ s = b.get("s", {}).get("value") if isinstance(b.get("s"), dict) else b.get("s")+ p = b.get("p", {}).get("value") if isinstance(b.get("p"), dict) else b.get("p")+ o = b.get("o", {}).get("value") if isinstance(b.get("o"), dict) else b.get("o")++ if s and p and o:+ triplets.append(Triplets(s, p, o))++ return triplets++ added_triples = extract_triplets(added_res.bindings)+ removed_triples = extract_triplets(removed_res.bindings)++ self.progress_tracker.stop_tracking(+ tracking_id,+ status="completed",+ message=f"Delte computed: +{len(added_triples)} / -{len(removed_triples)}"+ )++ return {+ "old_graph_uri": old_graph_uri,+ "new_graph_uri": new_graph_uri,+ "added_triples": added_triples,+ "removed_triples": removed_triples,+ "added_count": len(added_triples),+ "removed_triples": len(removed_triples),+ }++ except Exception as e:+ self.logger.error(f"Failed to compute delta: {e}")+ self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))+ raise ProcessingError(f"Delta computation failed: {e}")++diff --git a/tests/pipeline/test_pipeline_comprehensive.py b/tests/pipeline/test_pipeline_comprehensive.pyindex a0b3f826..667f571f 100644--- a/tests/pipeline/test_pipeline_comprehensive.py+++ b/tests/pipeline/test_pipeline_comprehensive.py@@ -219,6 +219,67 @@ class TestPipelineComprehensive(unittest.TestCase):self.assertTrue(result.success)self.assertEqual(result.output, "Success")self.assertEqual(mock_handler.call_count, 3)++ def test_execution_engine_delta_mode(self):+ """+ Test pipeline execution intercepting and computing delta mode.+ """++ mock_version_manager = MagicMock()+ mock_version_manager.get_version.side_effect = lambda v: {+ "v1": {"version_id": "v1", "graph_uri": "urn:graph:v1"},+ "v2": {"version_id": "v2", "graph_uri": "urn:graph:v2"}+ }.get(v)++ mock_triplet_store = MagicMock()+ expected_delta_payload = {+ "old_graph_uri": "urn:graph:v1",+ "new_graph_uri": "urn:graph:v2",+ "added_triples": ["<urn:s> <urn:p> <urn:o>"],+ "removed_triples": [],+ "added_count": 1,+ "removed_count": 0,+ }+ mock_triplet_store.compute_delta.return_value = expected_delta_payload+ def delta_aware_handler(data, **kwargs):+ return data++ builder = PipelineBuilder()+ builder.add_step(+ step_name="incremental_validation",+ step_type="validation",+ handler=delta_aware_handler,+ delta_mode=True,+ base_version_id="v1",+ target_version_id="v2",+ )++ pipeline = builder.build("delta_pipeline")+ engine = ExecutionEngine()++ initial_data = {"full_graph": "huge_amount_of_data"}++ result = engine.execute_pipeline(+ pipeline,+ data=initial_data,+ version_manager=mock_version_manager,+ triplet_store=mock_triplet_store+ )++ self.assertTrue(result.success)++ mock_version_manager.get_version.assert_any_call("v1")+ mock_version_manager.get_version.assert_any_call("v2")++ mock_triplet_store.compute_delta.assert_called_once_with(+ "urn:graph:v1",+ "urn:graph:v2",+ version_manager=mock_version_manager,+ triplet_store=mock_triplet_store+ )++ self.assertEqual(result.output, expected_delta_payload)+ self.assertNotEqual(result.output, initial_data)if __name__ == '__main__':unittest.main()