Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

feat: implement incremental delta processing

ZohaibHassan1659ff25fc0678
diff --git a/docs/reference/change_management.md b/docs/reference/change_management.md
index 0c2b24b2..012072f9 100644
--- a/docs/reference/change_management.md
+++ b/docs/reference/change_management.md
@@ -245,6 +245,49 @@ print(f"Axioms modified: {diff['axioms_modified']}")
---
+## Incremental / Delta processing
+
+For large-scale knowledge graphs, reprocessing the entire dataset on every update is computationally expensive.
+Semantica supports **Delta-Aware Pipelines**, allowing you to compute the exact differences (added and removed triples)
+between the two graph snapshots and run validation, enrichment, or export jobs *only* on the changes.
+
+### Delta Pipeline Example
+
+```python
+from semantica.change_management import TemporalVersionManager
+from semantica.pipeline import PipelineBuilder, ExecutionEngine
+
+# a. Initialize your managers
+version_manager = TemporalVersionManager(store_graph="kg_version.db")
+triplet_store = get_my_triplet_store()
+
+# b. Build a delta-aware pipeline
+builder = PipelineBuilder()
+builder.add_step(
+ step_name="validate_changes",
+ step_type="validation",
+ handler=my_validation_handler,
+ delta_mode=True, # Enables incremental processing
+ base_version_id="v1.0",
+ target_version_id="v1.1",
+)
+
+pipeline = builder.build("incremental_nightly_job")
+
+# c. Execute the pipeline
+engine = ExecutionEngine()
+
+# The engine dynamically intercepts the flow, computes the delta on the
+# database backend, and passes ONLY the changed triples to the handler.
+result = engine.execute_pipeline(
+ pipeline,
+ data={}, # Is ignored in delta mode
+ version_manager=version_manager,
+ triplet_store=triplet_store
+)
+```
+---
+
## Data Integrity
### compute_checksum
diff --git a/docs/reference/pipeline.md b/docs/reference/pipeline.md
index 74eea1dd..ca73cef5 100644
--- a/docs/reference/pipeline.md
+++ b/docs/reference/pipeline.md
@@ -131,7 +131,7 @@ The **Pipeline Module** provides a robust orchestration engine for building, exe
### Types
- `Pipeline` — Pipeline definition dataclass
-- `PipelineStep` — Pipeline step definition dataclass
+- `PipelineStep` — Pipeline step definition dataclass, Supports `delta_mode` (bool), `base_version_id` (str), and `target_version_id` (str) for incremental processing.
- `StepStatus` — Enum: `pending`, `running`, `completed`, `failed`, `skipped`
- `ExecutionResult` — Execution result dataclass
- `PipelineStatus` — Enum: `pending`, `running`, `paused`, `completed`, `failed`, `stopped`
@@ -405,6 +405,47 @@ result = engine.execute_pipeline(pipeline, data={"path": "document.pdf"})
---
+### Incremental / Delta-Aware Pipeline
+
+Use `delta_mode` to process only the differences between two graph versions, drastically reducing compute costs for large datasets.
+
+```python
+from semantica.pipeline import PipelineBuilder, ExecutionEngine
+
+builder = (
+ PipelineBuilder()
+ # Adding delta_mode=True tells the execution engine to intercept this step,
+ # compute the diff between v1 and v2, and pass ONLY the delta payload to the handler.
+ .add_step(
+ "validate_diff",
+ "validation",
+ delta_mode=True,
+ base_version_id="v1",
+ target_version_id="v2",
+ handler=diff_validator
+ )
+ .add_step(
+ "alert_on_removals",
+ "alerting",
+ dependencies=["validate_diff"],
+ handler=alert_handler
+ )
+)
+
+pipeline = builder.build(name="IncrementalJob")
+engine = ExecutionEngine()
+
+# Execution requires version_manager and triplet_store injected via options
+# so the engine can resolve URIs and compute the graph differences natively.
+result = engine.execute_pipeline(
+ pipeline,
+ version_manager=my_version_manager,
+ triplet_store=my_triplet_store
+)
+```
+
+---
+
## Best Practices
1. **Idempotency**: Ensure steps are idempotent (can be run multiple times without side effects) to support retries.
diff --git a/semantica/change_management/managers.py b/semantica/change_management/managers.py
index 6941c837..0dc24d04 100644
--- a/semantica/change_management/managers.py
+++ b/semantica/change_management/managers.py
@@ -6,7 +6,7 @@ and ontologies, with comprehensive change tracking, persistent storage, and audi
Key Features:
- Enhanced TemporalVersionManager for knowledge graphs
- - Enhanced VersionManager for ontologies
+ - Enhanced VersionManager for ontologies
- Detailed diff algorithms for entities and relationships
- Structural comparison for ontology elements
- Integration with storage backends and metadata
@@ -24,7 +24,13 @@ from datetime import datetime
from typing import Any, Dict, List, Optional
from .change_log import ChangeLogEntry
-from .version_storage import VersionStorage, InMemoryVersionStorage, SQLiteVersionStorage, compute_checksum, verify_checksum
+from .version_storage import (
+ VersionStorage,
+ InMemoryVersionStorage,
+ SQLiteVersionStorage,
+ compute_checksum,
+ verify_checksum,
+)
from ..utils.exceptions import ValidationError, ProcessingError
from ..utils.logging import get_logger
@@ -32,20 +38,20 @@ from ..utils.logging import get_logger
class BaseVersionManager(ABC):
"""
Abstract base class for enhanced version managers.
-
+
Provides common functionality for version management across different data types.
"""
-
+
def __init__(self, storage_path: Optional[str] = None):
"""
Initialize base version manager.
-
+
Args:
storage_path: Path to SQLite database for persistent storage.
If None, uses in-memory storage.
"""
self.logger = get_logger(self.__class__.__name__.lower())
-
+
# Initialize storage backend
if storage_path:
self.storage = SQLiteVersionStorage(storage_path)
@@ -53,25 +59,29 @@ class BaseVersionManager(ABC):
else:
self.storage = InMemoryVersionStorage()
self.logger.info("Initialized with in-memory storage")
-
+
@abstractmethod
- def create_snapshot(self, data: Any, version_label: str, author: str, description: str, **options) -> Dict[str, Any]:
+ def create_snapshot(
+ self, data: Any, version_label: str, author: str, description: str, **options
+ ) -> Dict[str, Any]:
"""Create a versioned snapshot of the data."""
pass
-
+
@abstractmethod
- def compare_versions(self, version1: Any, version2: Any, **options) -> Dict[str, Any]:
+ def compare_versions(
+ self, version1: Any, version2: Any, **options
+ ) -> Dict[str, Any]:
"""Compare two versions and return detailed differences."""
pass
-
+
def list_versions(self) -> List[Dict[str, Any]]:
"""List all version snapshots."""
return self.storage.list_all()
-
+
def get_version(self, label: str) -> Optional[Dict[str, Any]]:
"""Retrieve specific version by label."""
return self.storage.get(label)
-
+
def verify_checksum(self, snapshot: Dict[str, Any]) -> bool:
"""Verify the integrity of a snapshot using its checksum."""
return verify_checksum(snapshot)
@@ -80,10 +90,10 @@ class BaseVersionManager(ABC):
class TemporalVersionManager(BaseVersionManager):
"""
Temporal version management engine for knowledge graphs.
-
+
Provides comprehensive version/snapshot management capabilities including
persistent storage, detailed change tracking, and audit trails.
-
+
Features:
- Persistent snapshot storage (SQLite or in-memory)
- Detailed change tracking with entity-level diffs
@@ -92,11 +102,11 @@ class TemporalVersionManager(BaseVersionManager):
- Version comparison with backward compatibility
- Input validation and security features
"""
-
+
def __init__(self, storage_path: Optional[str] = None, **config):
"""
Initialize enhanced temporal version manager.
-
+
Args:
storage_path: Path to SQLite database file for persistent storage.
If None, uses in-memory storage
@@ -104,39 +114,37 @@ class TemporalVersionManager(BaseVersionManager):
"""
super().__init__(storage_path)
self.config = config
-
+
def create_snapshot(
- self,
- graph: Dict[str, Any],
- version_label: str,
- author: str,
+ self,
+ graph: Dict[str, Any],
+ version_label: str,
+ author: str,
description: str,
- **options
+ **options,
) -> Dict[str, Any]:
"""
Create and store snapshot with checksum and metadata.
-
+
Args:
graph: Knowledge graph dict with "entities" and "relationships"
version_label: Version string (e.g., "v1.0")
author: Email address of the change author
description: Change description (max 500 chars)
**options: Additional options
-
+
Returns:
dict: Snapshot with metadata and checksum
-
+
Raises:
ValidationError: If input validation fails
ProcessingError: If storage operation fails
"""
# Validate inputs
change_entry = ChangeLogEntry(
- timestamp=datetime.now().isoformat(),
- author=author,
- description=description
+ timestamp=datetime.now().isoformat(), author=author, description=description
)
-
+
# Create snapshot
snapshot = {
"label": version_label,
@@ -145,18 +153,18 @@ class TemporalVersionManager(BaseVersionManager):
"description": change_entry.description,
"entities": graph.get("entities", []).copy(),
"relationships": graph.get("relationships", []).copy(),
- "metadata": options.get("metadata", {})
+ "metadata": options.get("metadata", {}),
}
-
+
# Compute and add checksum
snapshot["checksum"] = compute_checksum(snapshot)
-
+
# Store snapshot
self.storage.save(snapshot)
-
+
self.logger.info(f"Created snapshot '{version_label}' by {author}")
return snapshot
-
+
def compare_versions(
self,
v1_label_or_dict,
@@ -166,13 +174,13 @@ class TemporalVersionManager(BaseVersionManager):
) -> Dict[str, Any]:
"""
Compare two graph versions with detailed entity-level differences.
-
+
Args:
v1_label_or_dict: First version (label string or snapshot dict)
v2_label_or_dict: Second version (label string or snapshot dict)
comparison_metrics: List of metrics to calculate (optional, unused)
**options: Additional comparison options (unused)
-
+
Returns:
dict: Detailed version comparison results
"""
@@ -183,17 +191,17 @@ class TemporalVersionManager(BaseVersionManager):
raise ValidationError(f"Version not found: {v1_label_or_dict}")
else:
version1 = v1_label_or_dict
-
+
if isinstance(v2_label_or_dict, str):
version2 = self.storage.get(v2_label_or_dict)
if not version2:
raise ValidationError(f"Version not found: {v2_label_or_dict}")
else:
version2 = v2_label_or_dict
-
+
# Compute detailed diff
detailed_diff = self._compute_detailed_diff(version1, version2)
-
+
# Maintain backward compatibility with summary
summary = {
"entities_added": len(detailed_diff["entities_added"]),
@@ -201,132 +209,197 @@ class TemporalVersionManager(BaseVersionManager):
"entities_modified": len(detailed_diff["entities_modified"]),
"relationships_added": len(detailed_diff["relationships_added"]),
"relationships_removed": len(detailed_diff["relationships_removed"]),
- "relationships_modified": len(detailed_diff["relationships_modified"])
+ "relationships_modified": len(detailed_diff["relationships_modified"]),
}
-
+
return {
"version1": version1.get("label", "unknown"),
"version2": version2.get("label", "unknown"),
"summary": summary,
- **detailed_diff
+ **detailed_diff,
}
-
- def _compute_detailed_diff(self, version1: Dict[str, Any], version2: Dict[str, Any]) -> Dict[str, Any]:
+
+ def _compute_detailed_diff(
+ self, version1: Dict[str, Any], version2: Dict[str, Any]
+ ) -> Dict[str, Any]:
"""
Compute detailed entity and relationship differences between versions.
-
+
Args:
version1: First version snapshot
version2: Second version snapshot
-
+
Returns:
Dict with detailed diff information
"""
- entities1 = {e.get("id", str(i)): e for i, e in enumerate(version1.get("entities", []))}
- entities2 = {e.get("id", str(i)): e for i, e in enumerate(version2.get("entities", []))}
-
- relationships1 = {self._relationship_key(r): r for r in version1.get("relationships", [])}
- relationships2 = {self._relationship_key(r): r for r in version2.get("relationships", [])}
-
+ entities1 = {
+ e.get("id", str(i)): e for i, e in enumerate(version1.get("entities", []))
+ }
+ entities2 = {
+ e.get("id", str(i)): e for i, e in enumerate(version2.get("entities", []))
+ }
+
+ relationships1 = {
+ self._relationship_key(r): r for r in version1.get("relationships", [])
+ }
+ relationships2 = {
+ self._relationship_key(r): r for r in version2.get("relationships", [])
+ }
+
# Entity differences
entity_ids1 = set(entities1.keys())
entity_ids2 = set(entities2.keys())
-
+
entities_added = [entities2[eid] for eid in entity_ids2 - entity_ids1]
entities_removed = [entities1[eid] for eid in entity_ids1 - entity_ids2]
-
+
entities_modified = []
for eid in entity_ids1 & entity_ids2:
if entities1[eid] != entities2[eid]:
changes = self._compute_entity_changes(entities1[eid], entities2[eid])
- entities_modified.append({
- "id": eid,
- "before": entities1[eid],
- "after": entities2[eid],
- "changes": changes
- })
-
+ entities_modified.append(
+ {
+ "id": eid,
+ "before": entities1[eid],
+ "after": entities2[eid],
+ "changes": changes,
+ }
+ )
+
# Relationship differences
rel_keys1 = set(relationships1.keys())
rel_keys2 = set(relationships2.keys())
-
+
relationships_added = [relationships2[key] for key in rel_keys2 - rel_keys1]
relationships_removed = [relationships1[key] for key in rel_keys1 - rel_keys2]
-
+
relationships_modified = []
for key in rel_keys1 & rel_keys2:
if relationships1[key] != relationships2[key]:
- changes = self._compute_relationship_changes(relationships1[key], relationships2[key])
- relationships_modified.append({
- "key": key,
- "before": relationships1[key],
- "after": relationships2[key],
- "changes": changes
- })
-
+ changes = self._compute_relationship_changes(
+ relationships1[key], relationships2[key]
+ )
+ relationships_modified.append(
+ {
+ "key": key,
+ "before": relationships1[key],
+ "after": relationships2[key],
+ "changes": changes,
+ }
+ )
+
return {
"entities_added": entities_added,
"entities_removed": entities_removed,
"entities_modified": entities_modified,
"relationships_added": relationships_added,
"relationships_removed": relationships_removed,
- "relationships_modified": relationships_modified
+ "relationships_modified": relationships_modified,
}
-
+
def _relationship_key(self, relationship: Dict[str, Any]) -> str:
"""Generate a unique key for a relationship."""
source = relationship.get("source", "")
target = relationship.get("target", "")
rel_type = relationship.get("type", relationship.get("relationship", ""))
return f"{source}|{rel_type}|{target}"
-
- def _compute_entity_changes(self, entity1: Dict[str, Any], entity2: Dict[str, Any]) -> Dict[str, Any]:
+
+ def _compute_entity_changes(
+ self, entity1: Dict[str, Any], entity2: Dict[str, Any]
+ ) -> Dict[str, Any]:
"""Compute changes between two entity versions."""
changes = {}
all_keys = set(entity1.keys()) | set(entity2.keys())
-
+
for key in all_keys:
val1 = entity1.get(key)
val2 = entity2.get(key)
-
+
if val1 != val2:
changes[key] = {"from": val1, "to": val2}
-
+
return changes
-
- def _compute_relationship_changes(self, rel1: Dict[str, Any], rel2: Dict[str, Any]) -> Dict[str, Any]:
+
+ def _compute_relationship_changes(
+ self, rel1: Dict[str, Any], rel2: Dict[str, Any]
+ ) -> Dict[str, Any]:
"""Compute changes between two relationship versions."""
changes = {}
all_keys = set(rel1.keys()) | set(rel2.keys())
-
+
for key in all_keys:
val1 = rel1.get(key)
val2 = rel2.get(key)
-
+
if val1 != val2:
changes[key] = {"from": val1, "to": val2}
-
+
return changes
+
+ def prune_versions(self, keep_last_n: int = 5, triplet_store: Any = None) -> Dict [str, Any]:
+ """
+ Prune old snapshots, keeping only the most recent N versions.
+ Optionally deletes the backend graphs from the triplet store to free space.
+
+ Args:
+ keep_last_n: Number of recent versions to retain.
+ triplet_store: Optional TripletScore instance to execute DROP GRAPH.
+
+ Returns:
+ Dict containing counts and labels of pruned versions.
+ """
+
+ all_versions = self.list_versions()
+ all_versions.sort(key = lambda x: x.get("timestamp", ""), reverse=True)
+
+ versions_to_delete = all_versions[keep_last_n:]
+ deleted_labels = []
+
+ for v in versions_to_delete:
+ label = v.get("label")
+ graph_uri = v.get("graph_uri")
+
+ # delete metadata from SQLite / In-Memory
+ if self.storage.delete(label):
+ deleted_labels.append(label)
+
+ # Clean up the actual graph if provided
+ if triplet_store and graph_uri:
+ try:
+ triplet_store.execute_query(f"DROP SILENT GRAPH {graph_uri}")
+ self.logger.info(f"Dropped obsolete graph {graph_uri} from store")
+ except Exception as e:
+ self.logger.warning(f"Failed to drop graph {graph_uri} during pruning: {e}")
+
+ self.logger.info(f"Pruned {len(deleted_labels)} old versions, kept {keep_last_n}")
+ return {
+ "pruned_count": len(deleted_labels),
+ "pruned_versions": deleted_labels,
+ "retained_count": len(all_versions) - len(deleted_labels)
+ }
+
+
+
class OntologyVersionManager(BaseVersionManager):
"""
Version management for ontologies with structural comparison.
-
+
Provides comprehensive version management for ontologies including
detailed structural analysis and change tracking.
-
+
Features:
- Structural comparison of ontology elements
- Detailed diff for classes, properties, individuals, axioms
- Persistent storage with metadata
- Change tracking and audit trails
"""
-
+
def __init__(self, storage_path: Optional[str] = None, **config):
"""
Initialize enhanced version manager.
-
+
Args:
storage_path: Path to SQLite database file for persistent storage.
If None, uses in-memory storage
@@ -335,35 +408,33 @@ class OntologyVersionManager(BaseVersionManager):
super().__init__(storage_path)
self.config = config
self.versions = {} # In-memory version tracking for compatibility
-
+
def create_snapshot(
self,
ontology_data: Dict[str, Any],
version_label: str,
author: str,
description: str,
- **options
+ **options,
) -> Dict[str, Any]:
"""
Create ontology version snapshot.
-
+
Args:
ontology_data: Ontology data dictionary
version_label: Version string (e.g., "v1.0")
author: Email address of the change author
description: Change description
**options: Additional options including metadata
-
+
Returns:
dict: Ontology version snapshot
"""
# Validate inputs
change_entry = ChangeLogEntry(
- timestamp=datetime.now().isoformat(),
- author=author,
- description=description
+ timestamp=datetime.now().isoformat(), author=author, description=description
)
-
+
# Create snapshot
snapshot = {
"label": version_label,
@@ -373,108 +444,112 @@ class OntologyVersionManager(BaseVersionManager):
"ontology_iri": ontology_data.get("uri", ""),
"version_info": ontology_data.get("version_info", {}),
"structure": ontology_data.get("structure", {}),
- "metadata": options.get("metadata", {})
+ "metadata": options.get("metadata", {}),
}
-
+
# Compute and add checksum
snapshot["checksum"] = compute_checksum(snapshot)
-
+
# Store snapshot
self.storage.save(snapshot)
-
+
# Also store in memory for compatibility
self.versions[version_label] = snapshot
-
+
self.logger.info(f"Created ontology snapshot '{version_label}' by {author}")
return snapshot
-
- def compare_versions(self, version1: str, version2: str, **options) -> Dict[str, Any]:
+
+ def compare_versions(
+ self, version1: str, version2: str, **options
+ ) -> Dict[str, Any]:
"""
Compare two ontology versions with detailed structural analysis.
-
+
Args:
version1: First version label
version2: Second version label
**options: Additional comparison options
-
+
Returns:
Detailed comparison results including structural differences
"""
# Get versions from storage
v1_snapshot = self.storage.get(version1)
v2_snapshot = self.storage.get(version2)
-
+
if not v1_snapshot:
raise ValidationError(f"Version not found: {version1}")
if not v2_snapshot:
raise ValidationError(f"Version not found: {version2}")
-
+
# Basic metadata comparison
metadata_changes = {}
if v1_snapshot.get("ontology_iri") != v2_snapshot.get("ontology_iri"):
metadata_changes["ontology_iri"] = {
"from": v1_snapshot.get("ontology_iri"),
- "to": v2_snapshot.get("ontology_iri")
+ "to": v2_snapshot.get("ontology_iri"),
}
if v1_snapshot.get("version_info") != v2_snapshot.get("version_info"):
metadata_changes["version_info"] = {
"from": v1_snapshot.get("version_info"),
- "to": v2_snapshot.get("version_info")
+ "to": v2_snapshot.get("version_info"),
}
-
+
# Structural comparison
structural_diff = self._compare_ontology_structures(v1_snapshot, v2_snapshot)
-
+
return {
"version1": version1,
"version2": version2,
"metadata_changes": metadata_changes,
- **structural_diff
+ **structural_diff,
}
-
- def _compare_ontology_structures(self, v1_snapshot: Dict[str, Any], v2_snapshot: Dict[str, Any]) -> Dict[str, Any]:
+
+ def _compare_ontology_structures(
+ self, v1_snapshot: Dict[str, Any], v2_snapshot: Dict[str, Any]
+ ) -> Dict[str, Any]:
"""
Compare structural elements between two ontology versions.
-
+
Args:
v1_snapshot: First ontology version snapshot
v2_snapshot: Second ontology version snapshot
-
+
Returns:
Dictionary with structural differences
"""
# Extract structural information
v1_structure = v1_snapshot.get("structure", {})
v2_structure = v2_snapshot.get("structure", {})
-
+
# Compare classes
v1_classes = set(v1_structure.get("classes", []))
v2_classes = set(v2_structure.get("classes", []))
-
+
classes_added = list(v2_classes - v1_classes)
classes_removed = list(v1_classes - v2_classes)
-
+
# Compare properties
v1_properties = set(v1_structure.get("properties", []))
v2_properties = set(v2_structure.get("properties", []))
-
+
properties_added = list(v2_properties - v1_properties)
properties_removed = list(v1_properties - v2_properties)
-
+
# Compare individuals
v1_individuals = set(v1_structure.get("individuals", []))
v2_individuals = set(v2_structure.get("individuals", []))
-
+
individuals_added = list(v2_individuals - v1_individuals)
individuals_removed = list(v1_individuals - v2_individuals)
-
+
# Compare axioms/rules
v1_axioms = set(v1_structure.get("axioms", []))
v2_axioms = set(v2_structure.get("axioms", []))
-
+
axioms_added = list(v2_axioms - v1_axioms)
axioms_removed = list(v1_axioms - v2_axioms)
-
+
return {
"classes_added": classes_added,
"classes_removed": classes_removed,
@@ -492,6 +567,6 @@ class OntologyVersionManager(BaseVersionManager):
"individuals_added": len(individuals_added),
"individuals_removed": len(individuals_removed),
"axioms_added": len(axioms_added),
- "axioms_removed": len(axioms_removed)
- }
+ "axioms_removed": len(axioms_removed),
+ },
}
diff --git a/semantica/change_management/version_storage.py b/semantica/change_management/version_storage.py
index aa2ee87b..b5e7e4b4 100644
--- a/semantica/change_management/version_storage.py
+++ b/semantica/change_management/version_storage.py
@@ -39,72 +39,105 @@ from ..utils.exceptions import ProcessingError, ValidationError
from ..utils.logging import get_logger
+
+def create_graph_snapshot_record(
+ version_id: str,
+ graph_uri: str,
+ author: str = "system",
+ description: str = "Graph snapshot",
+ metadata: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]:
+ """
+ Creates a standardized snapshot metadata record for a named graph.
+
+ Args:
+ version_id: Unique identifier for this snapshot
+ graph_uri: The underlying named graph URI in the triplet store
+ author: Creator of the snapshot
+ description: Purpose or context of the snapshot
+ metadata: Additional tags or piepline context
+ """
+
+ record = {
+ "label": version_id,
+ "version_id": version_id,
+ "graph_uri": graph_uri,
+ "timestamp": datetime.now().isoformat(),
+ "author": author,
+ "description": description,
+ "metadata": metadata or {},
+ }
+
+ record["checksum"] = compute_checksum(record)
+ return record
+
+
class VersionStorage(ABC):
"""
Abstract base class for version storage backends.
-
+
This interface defines the contract that all storage implementations
must follow for version management operations.
"""
-
+
@abstractmethod
def save(self, snapshot: Dict[str, Any]) -> None:
"""
Save a version snapshot.
-
+
Args:
snapshot: Version snapshot dictionary with metadata
-
+
Raises:
ValidationError: If snapshot data is invalid
ProcessingError: If save operation fails
"""
pass
-
+
@abstractmethod
def get(self, label: str) -> Optional[Dict[str, Any]]:
"""
Retrieve a version snapshot by label.
-
+
Args:
label: Version label to retrieve
-
+
Returns:
Snapshot dictionary or None if not found
"""
pass
-
+
@abstractmethod
def list_all(self) -> List[Dict[str, Any]]:
"""
List all version snapshots.
-
+
Returns:
List of snapshot metadata dictionaries
"""
pass
-
+
@abstractmethod
def exists(self, label: str) -> bool:
"""
Check if a version exists.
-
+
Args:
label: Version label to check
-
+
Returns:
True if version exists, False otherwise
"""
pass
-
+
@abstractmethod
def delete(self, label: str) -> bool:
"""
Delete a version snapshot.
-
+
Args:
label: Version label to delete
-
+
Returns:
True if deleted, False if not found
"""
@@ -114,31 +147,31 @@ class VersionStorage(ABC):
class InMemoryVersionStorage(VersionStorage):
"""
In-memory version storage implementation.
-
+
This implementation stores all version data in memory using a dictionary.
Data is lost when the process ends.
"""
-
+
def __init__(self):
"""Initialize in-memory storage."""
self._storage: Dict[str, Dict[str, Any]] = {}
self._lock = threading.RLock()
self.logger = get_logger("in_memory_storage")
-
+
def save(self, snapshot: Dict[str, Any]) -> None:
"""Save snapshot to memory."""
label = snapshot.get("label")
if not label:
raise ValidationError("Snapshot must have a 'label' field")
-
+
with self._lock:
if label in self._storage:
raise ValidationError(f"Version '{label}' already exists")
-
+
# Deep copy to prevent external modifications
self._storage[label] = json.loads(json.dumps(snapshot))
self.logger.debug(f"Saved version '{label}' to memory")
-
+
def get(self, label: str) -> Optional[Dict[str, Any]]:
"""Retrieve snapshot from memory."""
with self._lock:
@@ -147,7 +180,7 @@ class InMemoryVersionStorage(VersionStorage):
# Return deep copy to prevent external modifications
return json.loads(json.dumps(snapshot))
return None
-
+
def list_all(self) -> List[Dict[str, Any]]:
"""List all snapshots in memory."""
with self._lock:
@@ -156,21 +189,23 @@ class InMemoryVersionStorage(VersionStorage):
for label, snapshot in self._storage.items():
metadata = {
"label": snapshot.get("label"),
+ "version_id": snapshot.get("version_id", snapshot.get("label")),
+ "graph_uri": snapshot.get("graph_uri"),
"timestamp": snapshot.get("timestamp"),
"author": snapshot.get("author"),
"description": snapshot.get("description"),
"checksum": snapshot.get("checksum"),
"entity_count": len(snapshot.get("entities", [])),
- "relationship_count": len(snapshot.get("relationships", []))
- }
+ "relationship_count": len(snapshot.get("relationships", [])),
+ }
metadata_list.append(metadata)
return metadata_list
-
+
def exists(self, label: str) -> bool:
"""Check if version exists in memory."""
with self._lock:
return label in self._storage
-
+
def delete(self, label: str) -> bool:
"""Delete version from memory."""
with self._lock:
@@ -184,28 +219,28 @@ class InMemoryVersionStorage(VersionStorage):
class SQLiteVersionStorage(VersionStorage):
"""
SQLite-based persistent version storage implementation.
-
+
This implementation stores version data in a SQLite database file,
providing persistence across process restarts.
"""
-
+
def __init__(self, storage_path: str):
"""
Initialize SQLite storage.
-
+
Args:
storage_path: Path to SQLite database file
"""
self.storage_path = Path(storage_path)
self._lock = threading.RLock()
self.logger = get_logger("sqlite_storage")
-
+
# Create directory if it doesn't exist
self.storage_path.parent.mkdir(parents=True, exist_ok=True)
-
+
# Initialize database
self._init_database()
-
+
def _init_database(self) -> None:
"""Initialize SQLite database schema."""
with self._lock:
@@ -227,67 +262,73 @@ class SQLiteVersionStorage(VersionStorage):
self.logger.debug(f"Initialized SQLite database at {self.storage_path}")
finally:
conn.close()
-
+
def save(self, snapshot: Dict[str, Any]) -> None:
"""Save snapshot to SQLite database."""
label = snapshot.get("label")
if not label:
raise ValidationError("Snapshot must have a 'label' field")
-
+
with self._lock:
conn = sqlite3.connect(str(self.storage_path))
try:
cursor = conn.cursor()
-
+
# Check if version already exists
cursor.execute("SELECT label FROM versions WHERE label = ?", (label,))
if cursor.fetchone():
raise ValidationError(f"Version '{label}' already exists")
-
+
# Insert new version
- cursor.execute("""
+ cursor.execute(
+ """
INSERT INTO versions
(label, timestamp, author, description, checksum, snapshot_data, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?)
- """, (
- label,
- snapshot.get("timestamp", ""),
- snapshot.get("author", ""),
- snapshot.get("description", ""),
- snapshot.get("checksum", ""),
- json.dumps(snapshot),
- datetime.now().isoformat()
- ))
-
+ """,
+ (
+ label,
+ snapshot.get("timestamp", ""),
+ snapshot.get("author", ""),
+ snapshot.get("description", ""),
+ snapshot.get("checksum", ""),
+ json.dumps(snapshot),
+ datetime.now().isoformat(),
+ ),
+ )
+
conn.commit()
self.logger.debug(f"Saved version '{label}' to SQLite database")
-
+
except sqlite3.Error as e:
raise ProcessingError(f"Failed to save version to database: {e}")
finally:
conn.close()
-
+
def get(self, label: str) -> Optional[Dict[str, Any]]:
"""Retrieve snapshot from SQLite database."""
with self._lock:
conn = sqlite3.connect(str(self.storage_path))
try:
cursor = conn.cursor()
- cursor.execute("""
+ cursor.execute(
+ """
SELECT snapshot_data FROM versions WHERE label = ?
- """, (label,))
-
+ """,
+ (label,),
+ )
+
row = cursor.fetchone()
if not row:
return None
-
+
return json.loads(row[0])
-
+
except sqlite3.Error as e:
raise ProcessingError(f"Failed to retrieve version from database: {e}")
finally:
conn.close()
-
+
def list_all(self) -> List[Dict[str, Any]]:
"""List all snapshots in SQLite database."""
with self._lock:
@@ -297,29 +338,31 @@ class SQLiteVersionStorage(VersionStorage):
cursor.execute("""
SELECT snapshot_data FROM versions ORDER BY timestamp DESC
""")
-
+
metadata_list = []
for row in cursor.fetchall():
snapshot = json.loads(row[0])
-
+
metadata = {
"label": snapshot.get("label"),
+ "version_id": snapshot.get("version_id", snapshot.get("label")),
+ "graph_uri": snapshot.get("graph_uri"),
"timestamp": snapshot.get("timestamp"),
"author": snapshot.get("author"),
"description": snapshot.get("description"),
"checksum": snapshot.get("checksum"),
"entity_count": len(snapshot.get("entities", [])),
- "relationship_count": len(snapshot.get("relationships", []))
+ "relationship_count": len(snapshot.get("relationships", [])),
}
metadata_list.append(metadata)
-
+
return metadata_list
-
+
except sqlite3.Error as e:
raise ProcessingError(f"Failed to list versions from database: {e}")
finally:
conn.close()
-
+
def exists(self, label: str) -> bool:
"""Check if version exists in SQLite database."""
with self._lock:
@@ -332,7 +375,7 @@ class SQLiteVersionStorage(VersionStorage):
raise ProcessingError(f"Failed to check version existence: {e}")
finally:
conn.close()
-
+
def delete(self, label: str) -> bool:
"""Delete version from SQLite database."""
with self._lock:
@@ -342,12 +385,12 @@ class SQLiteVersionStorage(VersionStorage):
cursor.execute("DELETE FROM versions WHERE label = ?", (label,))
deleted = cursor.rowcount > 0
conn.commit()
-
+
if deleted:
self.logger.debug(f"Deleted version '{label}' from SQLite database")
-
+
return deleted
-
+
except sqlite3.Error as e:
raise ProcessingError(f"Failed to delete version from database: {e}")
finally:
@@ -357,35 +400,35 @@ class SQLiteVersionStorage(VersionStorage):
def compute_checksum(data: Dict[str, Any]) -> str:
"""
Compute SHA-256 checksum for version data.
-
+
Args:
data: Dictionary containing version data
-
+
Returns:
SHA-256 checksum as hexadecimal string
"""
# Create a deterministic JSON representation
- json_str = json.dumps(data, sort_keys=True, separators=(',', ':'))
- return hashlib.sha256(json_str.encode('utf-8')).hexdigest()
+ json_str = json.dumps(data, sort_keys=True, separators=(",", ":"))
+ return hashlib.sha256(json_str.encode("utf-8")).hexdigest()
def verify_checksum(snapshot: Dict[str, Any]) -> bool:
"""
Verify the integrity of a snapshot using its checksum.
-
+
Args:
snapshot: Snapshot dictionary with checksum field
-
+
Returns:
True if checksum is valid, False otherwise
"""
stored_checksum = snapshot.get("checksum")
if not stored_checksum:
return False
-
+
# Create copy without checksum for verification
data_copy = snapshot.copy()
data_copy.pop("checksum", None)
-
+
computed_checksum = compute_checksum(data_copy)
return stored_checksum == computed_checksum
diff --git a/semantica/pipeline/execution_engine.py b/semantica/pipeline/execution_engine.py
index 2cef1a37..16485b00 100644
--- a/semantica/pipeline/execution_engine.py
+++ b/semantica/pipeline/execution_engine.py
@@ -141,26 +141,37 @@ class ExecutionEngine:
# Register pipeline modules for progress tracking
module_list = []
module_order = {}
- if hasattr(pipeline, 'steps') and pipeline.steps:
+ if hasattr(pipeline, "steps") and pipeline.steps:
for idx, step in enumerate(pipeline.steps):
# Extract module name from step
- module_name = getattr(step, 'module', None) or getattr(step, 'name', None) or str(step)
+ module_name = (
+ getattr(step, "module", None)
+ or getattr(step, "name", None)
+ or str(step)
+ )
if module_name and module_name not in module_list:
module_list.append(module_name)
module_order[module_name] = idx
-
+
# If no steps found, try to infer from pipeline structure
if not module_list:
# Common pipeline modules
- module_list = ["ingest", "parse", "normalize", "semantic_extract", "kg", "embeddings"]
+ module_list = [
+ "ingest",
+ "parse",
+ "normalize",
+ "semantic_extract",
+ "kg",
+ "embeddings",
+ ]
module_order = {module: idx for idx, module in enumerate(module_list)}
-
+
# Register pipeline modules
if module_list:
self.progress_tracker.register_pipeline_modules(
pipeline_id=pipeline_id,
module_list=module_list,
- module_order=module_order
+ module_order=module_order,
)
# Set status
@@ -200,7 +211,7 @@ class ExecutionEngine:
status="completed" if metrics["steps_failed"] == 0 else "failed",
message=f"Executed {metrics['steps_executed']} steps in {execution_time:.2f}s",
)
-
+
# Clear pipeline context when pipeline completes
self.progress_tracker.clear_pipeline_context(pipeline_id)
@@ -300,11 +311,58 @@ class ExecutionEngine:
return current_data
def _execute_step(self, step: PipelineStep, data: Any, **options) -> Any:
- """Execute a single step."""
+ """
+ Execute a single step.
+
+ If delta_mode is enabled for the step, this intercepts the execution to compute
+ the delta between the base and target versions, passing only the changes
+ (added/removed triples) to the handler.
+ """
+
+ if getattr(step, "delta_mode", False):
+ self.logger.info(f"Executing step '{step.name}' in incremental delta mode.")
+
+ version_manager = options.get("version_manager") or self.config.get("version_manager")
+ triplet_store = options.get("triplet_store") or self.config.get("triplet_store")
+
+ if not version_manager or not triplet_store:
+ raise ProcessingError(
+ f"Step '{step.name}' requires 'version_manager' and 'triplet_store' "
+ f"in execution options for delta processing."
+ )
+
+ if not step.base_version_id or not step.target_version_id:
+ raise ValidationError(
+ f"Step '{step.name}' in delta_mode requires 'base_version_id' "
+ f"and 'target_version_id' to be set."
+ )
+
+
+ base_snap = version_manager.get_version(step.base_version_id)
+ target_snap = version_manager.get_version(step.target_version_id)
+
+ if not base_snap:
+ raise ValidationError(f"Base version '{step.base_version_id}' not found in storage.")
+ if not target_snap:
+ raise ValidationError(f"Target version '{step.target_version_id}' not found in storage.")
+
+ base_uri = base_snap.get("graph_uri")
+ target_uri = target_snap.get("graph_uri")
+
+ if not base_uri or not target_uri:
+ raise ValidationError(
+ "Both base and target snapshots must contain a 'graph_uri' "
+ "to compute native store deltas."
+ )
+
+ self.logger.debug(f"Computing delta between {base_uri} and {target_uri}")
+ delta_result = triplet_store.compute_delta(base_uri, target_uri, **options)
+
+ data = delta_result
+
if step.handler:
return step.handler(data, **step.config, **options)
else:
- # Default: pass data through
return data
def _topological_sort(self, steps: List[PipelineStep]) -> List[PipelineStep]:
@@ -406,9 +464,9 @@ class ExecutionEngine:
return {
"total_steps": total_steps,
"completed_steps": completed_steps,
- "progress_percentage": (completed_steps / total_steps * 100)
- if total_steps > 0
- else 0.0,
+ "progress_percentage": (
+ (completed_steps / total_steps * 100) if total_steps > 0 else 0.0
+ ),
"status": self.pipeline_status.get(
pipeline_id, PipelineStatus.PENDING
).value,
diff --git a/semantica/pipeline/pipeline_builder.py b/semantica/pipeline/pipeline_builder.py
index 22a2e21c..e4082e23 100644
--- a/semantica/pipeline/pipeline_builder.py
+++ b/semantica/pipeline/pipeline_builder.py
@@ -64,6 +64,9 @@ class PipelineStep:
status: StepStatus = StepStatus.PENDING
result: Any = None
error: Optional[Exception] = None
+ delta_mode: bool = False
+ base_version_id: Optional[str] = None
+ target_version_id: Optional[str] = None
@dataclass
@@ -125,16 +128,23 @@ class PipelineBuilder:
Returns:
Self for method chaining
"""
+ delta_mode = config.pop("delta_mode", False)
+ base_version_id = config.pop("base_version_id", None)
+ target_version_id = config.pop("target_version_id", None)
+
step = PipelineStep(
name=step_name,
step_type=step_type,
config=config,
dependencies=config.get("dependencies", []),
handler=config.get("handler"),
+ delta_mode = delta_mode,
+ base_version_id=base_version_id,
+ target_version_id=target_version_id,
)
self.steps.append(step)
- self.logger.debug(f"Added step: {step_name} ({step_type})")
+ self.logger.debug(f"Added step: {step_name} ({step_type}) | Delta Mode: {delta_mode}")
return self
@@ -397,6 +407,9 @@ class PipelineSerializer:
"type": step.step_type,
"config": step.config,
"dependencies": step.dependencies,
+ "delta_mode": getattr(step, "delta_mode", False),
+ "base_version_id": getattr(step, "base_version_id", None),
+ "target_version_id": getattr(step, "target_version_id", None),
}
for step in pipeline.steps
],
diff --git a/semantica/triplet_store/triplet_store.py b/semantica/triplet_store/triplet_store.py
index b689efdf..6ac1fc51 100644
--- a/semantica/triplet_store/triplet_store.py
+++ b/semantica/triplet_store/triplet_store.py
@@ -26,7 +26,7 @@ Author: Semantica Contributors
License: MIT
"""
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union, Triplets
from ..semantic_extract.triplet_extractor import Triplet
from ..utils.exceptions import ProcessingError, ValidationError
@@ -91,47 +91,44 @@ class TripletStore:
try:
if self.backend_type == "blazegraph":
from .blazegraph_store import BlazegraphStore
-
+
# Merge config with defaults
backend_config = self.config.copy()
if self.endpoint:
backend_config["endpoint"] = self.endpoint
else:
backend_config["endpoint"] = triplet_store_config.get(
- "blazegraph_endpoint",
- "http://localhost:9999/blazegraph"
+ "blazegraph_endpoint", "http://localhost:9999/blazegraph"
)
-
+
self._store_backend = BlazegraphStore(**backend_config)
elif self.backend_type == "jena":
from .jena_store import JenaStore
-
+
backend_config = self.config.copy()
if self.endpoint:
backend_config["endpoint"] = self.endpoint
else:
backend_config["endpoint"] = triplet_store_config.get(
- "jena_endpoint",
- "http://localhost:3030/ds"
+ "jena_endpoint", "http://localhost:3030/ds"
)
-
+
self._store_backend = JenaStore(**backend_config)
elif self.backend_type == "rdf4j":
from .rdf4j_store import RDF4JStore
-
+
backend_config = self.config.copy()
if self.endpoint:
backend_config["endpoint"] = self.endpoint
else:
backend_config["endpoint"] = triplet_store_config.get(
- "rdf4j_endpoint",
- "http://localhost:8080/rdf4j-server"
+ "rdf4j_endpoint", "http://localhost:8080/rdf4j-server"
)
-
+
self._store_backend = RDF4JStore(**backend_config)
-
+
self.logger.info(f"Initialized {self.backend_type} backend")
except Exception as e:
@@ -139,19 +136,19 @@ class TripletStore:
raise ProcessingError(f"Failed to initialize backend: {e}")
def store(
- self,
- knowledge_graph: Union[Dict[str, Any], Any],
- ontology: Union[Dict[str, Any], Any],
- **options
+ self,
+ knowledge_graph: Union[Dict[str, Any], Any],
+ ontology: Union[Dict[str, Any], Any],
+ **options,
) -> Dict[str, Any]:
"""
Store knowledge graph and ontology in the triplet store.
-
+
Args:
knowledge_graph: Knowledge graph dictionary or object
ontology: Ontology dictionary or object
**options: Additional options
-
+
Returns:
Operation status
"""
@@ -160,9 +157,9 @@ class TripletStore:
knowledge_graph = knowledge_graph.to_dict()
if hasattr(ontology, "to_dict"):
ontology = ontology.to_dict()
-
+
triplets = []
-
+
# Standard Namespaces
RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
RDFS_SUBCLASS = "http://www.w3.org/2000/01/rdf-schema#subClassOf"
@@ -171,23 +168,23 @@ class TripletStore:
OWL_DATATYPE_PROPERTY = "http://www.w3.org/2002/07/owl#DatatypeProperty"
RDFS_DOMAIN = "http://www.w3.org/2000/01/rdf-schema#domain"
RDFS_RANGE = "http://www.w3.org/2000/01/rdf-schema#range"
-
+
# 1. Process Ontology
classes = ontology.get("classes", [])
properties = ontology.get("properties", [])
-
+
for cls in classes:
# Class definition
cls_uri = cls.get("uri") or cls.get("id") or cls.get("name")
if not cls_uri:
continue
-
+
if not cls_uri.startswith("http") and not cls_uri.startswith("urn:"):
- # Fallback if no URI provided
- cls_uri = f"urn:class:{cls_uri}"
-
+ # Fallback if no URI provided
+ cls_uri = f"urn:class:{cls_uri}"
+
triplets.append(Triplet(cls_uri, RDF_TYPE, OWL_CLASS))
-
+
# Hierarchy
parent = cls.get("parent") or cls.get("subClassOf")
if parent:
@@ -195,15 +192,15 @@ class TripletStore:
if not parent.startswith("http") and not parent.startswith("urn:"):
parent_uri = f"urn:class:{parent}"
triplets.append(Triplet(cls_uri, RDFS_SUBCLASS, parent_uri))
-
+
for prop in properties:
prop_uri = prop.get("uri") or prop.get("id") or prop.get("name")
if not prop_uri:
continue
-
+
if not prop_uri.startswith("http") and not prop_uri.startswith("urn:"):
- prop_uri = f"urn:property:{prop_uri}"
-
+ prop_uri = f"urn:property:{prop_uri}"
+
# Determine property type (Object or Datatype)
# Default to ObjectProperty if not specified
prop_type = prop.get("type", OWL_OBJECT_PROPERTY)
@@ -211,25 +208,25 @@ class TripletStore:
prop_type = OWL_DATATYPE_PROPERTY
elif prop_type == "object":
prop_type = OWL_OBJECT_PROPERTY
-
+
triplets.append(Triplet(prop_uri, RDF_TYPE, prop_type))
-
+
if "domain" in prop:
domains = prop["domain"]
if isinstance(domains, str):
domains = [domains]
-
+
for domain in domains:
domain_uri = domain
if not domain.startswith("http") and not domain.startswith("urn:"):
domain_uri = f"urn:class:{domain}"
triplets.append(Triplet(prop_uri, RDFS_DOMAIN, domain_uri))
-
+
if "range" in prop:
ranges = prop["range"]
if isinstance(ranges, str):
ranges = [ranges]
-
+
for range_ in ranges:
range_uri = range_
if not range_.startswith("http") and not range_.startswith("urn:"):
@@ -239,28 +236,30 @@ class TripletStore:
# 2. Process Knowledge Graph
entities = knowledge_graph.get("entities", [])
relationships = knowledge_graph.get("relationships", [])
-
- entity_map = {} # Map IDs to URIs
-
+
+ entity_map = {} # Map IDs to URIs
+
for entity in entities:
entity_id = entity.get("id")
if not entity_id:
continue
-
+
entity_uri = entity.get("uri")
if not entity_uri:
entity_uri = f"urn:entity:{entity_id}"
-
+
entity_map[entity_id] = entity_uri
-
+
# Entity Type
entity_type = entity.get("type")
if entity_type:
type_uri = entity_type
- if not entity_type.startswith("http") and not entity_type.startswith("urn:"):
+ if not entity_type.startswith("http") and not entity_type.startswith(
+ "urn:"
+ ):
type_uri = f"urn:class:{entity_type}"
triplets.append(Triplet(entity_uri, RDF_TYPE, type_uri))
-
+
# Entity Properties
props = entity.get("properties", {})
for k, v in props.items():
@@ -268,23 +267,23 @@ class TripletStore:
if not k.startswith("http") and not k.startswith("urn:"):
prop_uri = f"urn:property:{k}"
triplets.append(Triplet(entity_uri, prop_uri, str(v)))
-
+
for rel in relationships:
source_id = rel.get("source")
target_id = rel.get("target")
rel_type = rel.get("type") or rel.get("label")
-
+
if not source_id or not target_id or not rel_type:
continue
-
+
source_uri = entity_map.get(source_id, f"urn:entity:{source_id}")
target_uri = entity_map.get(target_id, f"urn:entity:{target_id}")
rel_uri = rel_type
if not rel_type.startswith("http") and not rel_type.startswith("urn:"):
rel_uri = f"urn:property:{rel_type}"
-
+
triplets.append(Triplet(source_uri, rel_uri, target_uri))
-
+
# Bulk load all triplets
return self.add_triplets(triplets, **options)
@@ -305,10 +304,7 @@ class TripletStore:
return self._store_backend.add_triplet(triplet, **options)
def add_triplets(
- self,
- triplets: List[Triplet],
- batch_size: int = 1000,
- **options
+ self, triplets: List[Triplet], batch_size: int = 1000, **options
) -> Dict[str, Any]:
"""
Add multiple triplets to the store (bulk load).
@@ -330,10 +326,7 @@ class TripletStore:
# Use bulk loader for efficient processing
progress = self.bulk_loader.load_triplets(
- valid_triplets,
- self._store_backend,
- batch_size=batch_size,
- **options
+ valid_triplets, self._store_backend, batch_size=batch_size, **options
)
return {
@@ -341,7 +334,7 @@ class TripletStore:
"total": progress.total_triplets,
"processed": progress.loaded_triplets,
"failed": progress.failed_triplets,
- "batches": progress.total_batches
+ "batches": progress.total_batches,
}
def get_triplets(
@@ -364,10 +357,7 @@ class TripletStore:
List of matching Triplet objects
"""
return self._store_backend.get_triplets(
- subject=subject,
- predicate=predicate,
- object=object,
- **options
+ subject=subject, predicate=predicate, object=object, **options
)
def delete_triplet(self, triplet: Triplet, **options) -> Dict[str, Any]:
@@ -384,10 +374,7 @@ class TripletStore:
return self._store_backend.delete_triplet(triplet, **options)
def update_triplet(
- self,
- old_triplet: Triplet,
- new_triplet: Triplet,
- **options
+ self, old_triplet: Triplet, new_triplet: Triplet, **options
) -> Dict[str, Any]:
"""
Update a triplet (atomic delete + add).
@@ -406,10 +393,7 @@ class TripletStore:
return self.add_triplet(new_triplet, **options)
def execute_query(
- self,
- query: str,
- parameters: Optional[Dict[str, Any]] = None,
- **options
+ self, query: str, parameters: Optional[Dict[str, Any]] = None, **options
) -> Any:
"""
Execute a SPARQL query.
@@ -428,12 +412,14 @@ class TripletStore:
"""Validate triplet structure."""
if not triplet.subject or not triplet.predicate or not triplet.object:
return False
-
+
# Check confidence score if present
- if hasattr(triplet, 'confidence'):
- if triplet.confidence is not None and (triplet.confidence < 0 or triplet.confidence > 1):
+ if hasattr(triplet, "confidence"):
+ if triplet.confidence is not None and (
+ triplet.confidence < 0 or triplet.confidence > 1
+ ):
return False
-
+
return True
def get_stats(self) -> Dict[str, Any]:
@@ -441,3 +427,84 @@ class TripletStore:
if hasattr(self._store_backend, "get_stats"):
return self._store_backend.get_stats()
return {}
+
+ def compute_delta(
+ self, old_graph_uri: str, new_graph_uri: str, **options
+ ) -> Dict[str, Any]:
+ """
+ Compute the delta (added and removed triples) between two graph snapshots.
+
+ Args:
+ old_graph_uri: URI of the baseline graph snapshot.
+ new_graph_uri: URI of the target graph snapshot
+ **options: Additional query execution options
+
+ Returns:
+ Dictionary containing added_triples, removed_triples, and counts.
+ """
+
+ tracking_id = self.progress_tracker.start_tracking(
+ module="triplet_store",
+ submodule="COmputeDelta",
+ message=f"Computing delta: {old_graph_uri} -> {new_graph_uri}",
+ )
+
+ # SPARQL: Triples in the new graph that do not exist in the old graph
+ added_query = f"""
+ SELECT ?s ?p ?o WHERE {{
+ GRAPH <{new_graph_uri} > {{ ?s ?p ?o }}
+ FILTER NOT EXISTS {{ GRAPH <{old_graph_uri}> {{ ?s ?o ?p}} }}
+ }}
+ """
+
+ # // : Triples in the old graph that do not exist in the new graph
+ removed_query = f"""
+ SELECT ?s ?p ?o WHERE {{
+ GRAPH <{old_graph_uri}> {{ ?s ?p ?o }}
+ FILTER NOT EXISTS {{ GRAPH <{new_graph_uri}> {{ ?s ?p ?o }} }}
+ }}
+ """
+
+ try:
+ self.progress_tracker.update_tracking(tracking_id, message="Executing added triples query...")
+ added_res = self.execute_query(added_query, **options)
+
+ self.progress_tracker.update_tracking(tracking_id, message="Executing removed triples query...")
+ removed_res = self.execute_query(removed_query, **options)
+
+ def extract_triplets(bindings):
+ triplets = []
+ for b in bindings:
+ s = b.get("s", {}).get("value") if isinstance(b.get("s"), dict) else b.get("s")
+ p = b.get("p", {}).get("value") if isinstance(b.get("p"), dict) else b.get("p")
+ o = b.get("o", {}).get("value") if isinstance(b.get("o"), dict) else b.get("o")
+
+ if s and p and o:
+ triplets.append(Triplets(s, p, o))
+
+ return triplets
+
+ added_triples = extract_triplets(added_res.bindings)
+ removed_triples = extract_triplets(removed_res.bindings)
+
+ self.progress_tracker.stop_tracking(
+ tracking_id,
+ status="completed",
+ message=f"Delte computed: +{len(added_triples)} / -{len(removed_triples)}"
+ )
+
+ return {
+ "old_graph_uri": old_graph_uri,
+ "new_graph_uri": new_graph_uri,
+ "added_triples": added_triples,
+ "removed_triples": removed_triples,
+ "added_count": len(added_triples),
+ "removed_triples": len(removed_triples),
+ }
+
+ except Exception as e:
+ self.logger.error(f"Failed to compute delta: {e}")
+ self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
+ raise ProcessingError(f"Delta computation failed: {e}")
+
+
diff --git a/tests/pipeline/test_pipeline_comprehensive.py b/tests/pipeline/test_pipeline_comprehensive.py
index a0b3f826..667f571f 100644
--- a/tests/pipeline/test_pipeline_comprehensive.py
+++ b/tests/pipeline/test_pipeline_comprehensive.py
@@ -219,6 +219,67 @@ class TestPipelineComprehensive(unittest.TestCase):
self.assertTrue(result.success)
self.assertEqual(result.output, "Success")
self.assertEqual(mock_handler.call_count, 3)
+
+ def test_execution_engine_delta_mode(self):
+ """
+ Test pipeline execution intercepting and computing delta mode.
+ """
+
+ mock_version_manager = MagicMock()
+ mock_version_manager.get_version.side_effect = lambda v: {
+ "v1": {"version_id": "v1", "graph_uri": "urn:graph:v1"},
+ "v2": {"version_id": "v2", "graph_uri": "urn:graph:v2"}
+ }.get(v)
+
+ mock_triplet_store = MagicMock()
+ expected_delta_payload = {
+ "old_graph_uri": "urn:graph:v1",
+ "new_graph_uri": "urn:graph:v2",
+ "added_triples": ["<urn:s> <urn:p> <urn:o>"],
+ "removed_triples": [],
+ "added_count": 1,
+ "removed_count": 0,
+ }
+ mock_triplet_store.compute_delta.return_value = expected_delta_payload
+ def delta_aware_handler(data, **kwargs):
+ return data
+
+ builder = PipelineBuilder()
+ builder.add_step(
+ step_name="incremental_validation",
+ step_type="validation",
+ handler=delta_aware_handler,
+ delta_mode=True,
+ base_version_id="v1",
+ target_version_id="v2",
+ )
+
+ pipeline = builder.build("delta_pipeline")
+ engine = ExecutionEngine()
+
+ initial_data = {"full_graph": "huge_amount_of_data"}
+
+ result = engine.execute_pipeline(
+ pipeline,
+ data=initial_data,
+ version_manager=mock_version_manager,
+ triplet_store=mock_triplet_store
+ )
+
+ self.assertTrue(result.success)
+
+ mock_version_manager.get_version.assert_any_call("v1")
+ mock_version_manager.get_version.assert_any_call("v2")
+
+ mock_triplet_store.compute_delta.assert_called_once_with(
+ "urn:graph:v1",
+ "urn:graph:v2",
+ version_manager=mock_version_manager,
+ triplet_store=mock_triplet_store
+ )
+
+ self.assertEqual(result.output, expected_delta_payload)
+ self.assertNotEqual(result.output, initial_data)
if __name__ == '__main__':
unittest.main()