Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
feat(provenance): implement Global Default Storage pattern, CLI methods, and orchestrator config wiring
Sameer63052dd06aadcd93
diff --git a/semantica/cli.py b/semantica/cli.pyindex c9c362e6..c66d383f 100644--- a/semantica/cli.py+++ b/semantica/cli.py@@ -2535,6 +2535,9 @@ def provenance_audit(cli_ctx: CLIContext, since: Optional[str], fmt: str,cli_ctx = _require_ctx(cli_ctx)def _action() -> None:+ if _is_dry(cli_ctx, False):+ _dry(cli_ctx, "export audit log", since=since, format=fmt, output=output)+ returntry:from .provenance import ProvenanceManagerpm = ProvenanceManager(config=cli_ctx.config.to_dict())diff --git a/semantica/core/config_manager.py b/semantica/core/config_manager.pyindex 219bd157..1446c0bf 100644--- a/semantica/core/config_manager.py+++ b/semantica/core/config_manager.py@@ -51,7 +51,6 @@ Author: Semantica ContributorsLicense: MIT"""-import jsonimport osfrom pathlib import Pathfrom typing import Any, Dict, List, Optional, Union@@ -67,7 +66,6 @@ from ..utils.helpers import (set_nested_value,)from ..utils.progress_tracker import get_progress_tracker-from ..utils.validators import validate_configclass Config:@@ -157,6 +155,7 @@ class Config:self.security = config_data.get("security", DEFAULT_CONFIG.get("security", {}))+ self.provenance = config_data.get("provenance", {})self.custom = config_data.get("custom", {})def _load_from_env(self, config_dict: Dict[str, Any]) -> None:@@ -346,6 +345,7 @@ class Config:"logging": self.logging,"quality": self.quality,"security": self.security,+ "provenance": self.provenance,"custom": self.custom,}diff --git a/semantica/core/orchestrator.py b/semantica/core/orchestrator.pyindex e9d9a138..f0c8cfb2 100644--- a/semantica/core/orchestrator.py+++ b/semantica/core/orchestrator.py@@ -27,11 +27,11 @@ License: MITfrom pathlib import Pathfrom typing import Any, Dict, List, Optional, Union-from ..utils.exceptions import ConfigurationError, ProcessingError+from ..utils.exceptions import ProcessingErrorfrom ..utils.logging import get_logger, log_execution_timefrom ..utils.progress_tracker import get_progress_trackerfrom .config_manager import Config, ConfigManager-from .lifecycle import LifecycleManager, SystemState+from .lifecycle import LifecycleManagerfrom .plugin_registry import PluginRegistry@@ -92,7 +92,9 @@ class Semantica:# Configure global provenance storage path if specifiedtry:- prov_storage_path = self.config.get("provenance", {}).get("storage_path")+ prov_storage_path = self.config.get("provenance.storage_path")+ if not prov_storage_path and isinstance(self.config.get("provenance"), dict):+ prov_storage_path = self.config.get("provenance", {}).get("storage_path")if prov_storage_path:from ..provenance import ProvenanceManagerProvenanceManager.set_default_storage_path(prov_storage_path)@@ -640,10 +642,10 @@ class Semantica:try:# Import key modules to verify they're available# These imports don't create instances, just verify module availability- from ..ingest import FileIngestor- from ..kg import GraphBuilder- from ..parse import DocumentParser- from ..pipeline import PipelineBuilder+ from ..ingest import FileIngestor # noqa: F401+ from ..kg import GraphBuilder # noqa: F401+ from ..parse import DocumentParser # noqa: F401+ from ..pipeline import PipelineBuilder # noqa: F401self.logger.debug("Framework modules verified and available")except (ImportError, OSError) as e:diff --git a/semantica/provenance/__init__.py b/semantica/provenance/__init__.pyindex 706f804a..c6faaea4 100644--- a/semantica/provenance/__init__.py+++ b/semantica/provenance/__init__.py@@ -43,7 +43,7 @@ License: MITfrom .schemas import ProvenanceEntry, SourceReferencefrom .storage import ProvenanceStorage, InMemoryStorage, SQLiteStorage-from .manager import ProvenanceManager+from .manager import ProvenanceManager, default_storage_pathfrom .integrity import compute_checksum, verify_checksum__all__ = [@@ -58,6 +58,7 @@ __all__ = [# Manager"ProvenanceManager",+ "default_storage_path",# Utilities"compute_checksum",diff --git a/semantica/provenance/manager.py b/semantica/provenance/manager.pyindex 85b516ce..04414c02 100644--- a/semantica/provenance/manager.py+++ b/semantica/provenance/manager.py@@ -24,15 +24,30 @@ Author: Semantica ContributorsLicense: MIT"""-from typing import Optional, List, Dict, Any+from typing import Optional, List, Dict, Any, Unionfrom collections.abc import Mappingfrom datetime import datetime+from contextlib import contextmanager-from .schemas import ProvenanceEntry, SourceReference, PropertySource+from .schemas import ProvenanceEntry, SourceReferencefrom .storage import ProvenanceStorage, InMemoryStorage, SQLiteStoragefrom .integrity import compute_checksum+@contextmanager+def default_storage_path(path: Optional[str]):+ """+ Context manager for temporarily setting ProvenanceManager._default_storage_path.+ Guarantees restoration to the previous value on exit (safe for tests).+ """+ original_path = ProvenanceManager._default_storage_path+ ProvenanceManager._default_storage_path = path+ try:+ yield+ finally:+ ProvenanceManager._default_storage_path = original_path++class ProvenanceManager:"""Unified provenance tracking manager.@@ -57,15 +72,29 @@ class ProvenanceManager:_default_storage_path: Optional[str] = None@classmethod- def set_default_storage_path(cls, path: str) -> None:+ def set_default_storage_path(cls, path: Optional[str]) -> None:"""Set a global default storage path for all new instances."""cls._default_storage_path = path++ @classmethod+ @contextmanager+ def default_storage_path(cls, path: Optional[str]):+ """+ Context manager for temporarily setting the global default storage path.+ Guarantees restoration to the previous value on exit (safe for tests).+ """+ original_path = cls._default_storage_path+ cls._default_storage_path = path+ try:+ yield+ finally:+ cls._default_storage_path = original_pathdef __init__(self,storage: Optional[ProvenanceStorage] = None,storage_path: Optional[str] = None,- config: Optional[Dict[str, Any]] = None,+ config: Optional[Any] = None,**kwargs):"""@@ -74,14 +103,19 @@ class ProvenanceManager:Args:storage: Custom storage backend (optional)storage_path: Path to SQLite database (optional, uses in-memory if None)- config: Configuration dictionary (optional)+ config: Configuration dictionary or mapping (optional)"""if storage:self.storage = storagereturnif not storage_path and config:- storage_path = config.get("provenance", {}).get("storage_path")+ if isinstance(config, Mapping) or isinstance(config, dict):+ prov_config = config.get("provenance", {})+ if (isinstance(prov_config, Mapping) or isinstance(prov_config, dict)) and "storage_path" in prov_config:+ storage_path = prov_config.get("storage_path")+ elif "storage_path" in config:+ storage_path = config.get("storage_path")if not storage_path and self._default_storage_path:storage_path = self._default_storage_path@@ -626,3 +660,153 @@ class ProvenanceManager:if e.source_document))}++ # === CLI Integration Methods ===++ def lineage(self, entity_id: str, depth: int = 3) -> Dict[str, Any]:+ """+ Get lineage information formatted for CLI display.++ Args:+ entity_id: ID of the entity to trace lineage for+ depth: Maximum traversal depth (default: 3)++ Returns:+ Dict containing entity_id, depth, count, lineage entries, and sources+ """+ base_lineage = self.get_lineage(entity_id)+ entries = base_lineage.get("entries", [])+ if len(entries) > depth:+ entries = entries[:depth]+ sources = self.get_all_sources(entity_id)+ return {+ "entity_id": entity_id,+ "depth": depth,+ "chain_length": len(entries),+ "source_documents": base_lineage.get("source_documents", []),+ "lineage": entries,+ "entries": entries,+ "sources": sources,+ "metadata": base_lineage.get("metadata", {}),+ }++ def audit_log(+ self, since: Optional[str] = None, format: str = "table"+ ) -> Union[str, List[Dict[str, Any]]]:+ """+ Export audit log of provenance entries.++ Args:+ since: Optional ISO 8601 date string filter+ format: Output format ('table', 'csv', 'json')++ Returns:+ Formatted audit log as string or list of dicts+ """+ entries = self.storage.retrieve_all()+ if since:+ entries = [e for e in entries if getattr(e, "timestamp", "") >= since]+ entries.sort(key=lambda e: getattr(e, "timestamp", ""))++ if format == "json":+ return [+ e.to_dict() if hasattr(e, "to_dict") else getattr(e, "__dict__", {})+ for e in entries+ ]+ elif format == "csv":+ lines = ["entity_id,entity_type,activity_id,agent_id,timestamp"]+ for e in entries:+ lines.append(+ f"{e.entity_id},{e.entity_type},{e.activity_id},{e.agent_id},{getattr(e, 'timestamp', '')}"+ )+ return "\n".join(lines)+ else:+ lines = [+ f"{'ENTITY_ID':<20} {'TYPE':<15} {'ACTIVITY':<15} {'TIMESTAMP':<25}"+ ]+ lines.append("-" * 75)+ for e in entries:+ lines.append(+ f"{str(e.entity_id):<20} {str(e.entity_type):<15} {str(e.activity_id):<15} {str(getattr(e, 'timestamp', '')):<25}"+ )+ return "\n".join(lines)++ def export_prov(self, format: str = "turtle") -> str:+ """+ Export provenance as W3C PROV-O RDF.++ Args:+ format: RDF format ('turtle', 'ntriples', 'jsonld')++ Returns:+ Serialized RDF string+ """+ from rdflib import Graph, Literal, Namespace, URIRef+ from rdflib.namespace import RDF, XSD++ PROV = Namespace("http://www.w3.org/ns/prov#")+ EX = Namespace("http://example.org/ns/")++ g = Graph()+ g.bind("prov", PROV)+ g.bind("ex", EX)++ for e in self.storage.retrieve_all():+ ent_uri = URIRef(EX[str(e.entity_id)])+ g.add((ent_uri, RDF.type, PROV.Entity))++ if getattr(e, "timestamp", None):+ g.add(+ (+ ent_uri,+ PROV.generatedAtTime,+ Literal(e.timestamp, datatype=XSD.dateTime),+ )+ )++ if getattr(e, "agent_id", None) and e.agent_id != "unknown":+ ag_uri = URIRef(EX[str(e.agent_id)])+ g.add((ag_uri, RDF.type, PROV.Agent))+ g.add((ent_uri, PROV.wasAttributedTo, ag_uri))++ if getattr(e, "activity_id", None) and e.activity_id != "unknown":+ act_uri = URIRef(EX[str(e.activity_id)])+ g.add((act_uri, RDF.type, PROV.Activity))+ g.add((ent_uri, PROV.wasGeneratedBy, act_uri))++ for parent_id in getattr(e, "derived_from", []):+ p_uri = URIRef(EX[str(parent_id)])+ g.add((ent_uri, PROV.wasDerivedFrom, p_uri))++ rdf_format = "json-ld" if format == "jsonld" else format+ return g.serialize(format=rdf_format)++ def check(self, strict: bool = False) -> Dict[str, Any]:+ """+ Validate provenance integrity.++ Args:+ strict: Whether to perform strict validation++ Returns:+ Dictionary with validation results+ """+ entries = self.storage.retrieve_all()+ all_ids = {e.entity_id for e in entries}++ missing_refs = []+ for e in entries:+ for p in getattr(e, "derived_from", []):+ if p not in all_ids:+ missing_refs.append(f"{e.entity_id} -> {p}")++ valid = len(missing_refs) == 0 if strict else True++ return {+ "valid": valid,+ "total_entries": len(entries),+ "missing_references": missing_refs,+ "strict": strict,+ "errors": len(missing_refs) if strict else 0,+ }+diff --git a/tests/provenance/test_manager.py b/tests/provenance/test_manager.pyindex 6d704d8f..872277e5 100644--- a/tests/provenance/test_manager.py+++ b/tests/provenance/test_manager.py@@ -11,7 +11,89 @@ from semantica.provenance import ProvenanceManager, SourceReferenceclass TestProvenanceManager:"""Test ProvenanceManager functionality."""-++ @pytest.fixture(autouse=True)+ def reset_default_storage_path(self):+ original = ProvenanceManager._default_storage_path+ try:+ yield+ finally:+ ProvenanceManager._default_storage_path = original++ def test_default_storage_path_pattern(self, tmp_path):+ """Test global default storage pattern, config kwarg, and test-isolation context manager."""+ from semantica.provenance import default_storage_path, InMemoryStorage, SQLiteStorage++ # 1. Default should fallback to InMemoryStorage when no path/config+ prov_mgr = ProvenanceManager()+ assert isinstance(prov_mgr.storage, InMemoryStorage)++ # 2. Config kwarg should extract storage_path gracefully (CLI bug fix)+ cfg_path = str(tmp_path / "cfg_test.db")+ cfg = {"provenance": {"storage_path": cfg_path}}+ prov_mgr = ProvenanceManager(config=cfg)+ assert isinstance(prov_mgr.storage, SQLiteStorage)++ # 3. Explicit storage_path arg overrides config and default+ explicit_path = str(tmp_path / "explicit.db")+ prov_mgr = ProvenanceManager(storage_path=explicit_path, config=cfg)+ assert isinstance(prov_mgr.storage, SQLiteStorage)++ # 4. default_storage_path context manager should temporarily set default and restore on exit+ ctx_path = str(tmp_path / "ctx_test.db")+ with default_storage_path(ctx_path):+ assert ProvenanceManager._default_storage_path == ctx_path+ prov_mgr = ProvenanceManager()+ assert isinstance(prov_mgr.storage, SQLiteStorage)++ # Should be restored after context exit+ assert ProvenanceManager._default_storage_path is None+ prov_mgr = ProvenanceManager()+ assert isinstance(prov_mgr.storage, InMemoryStorage)++ # 5. Guaranteed restoration even on exception+ exc_path = str(tmp_path / "exception_test.db")+ try:+ with ProvenanceManager.default_storage_path(exc_path):+ assert ProvenanceManager._default_storage_path == exc_path+ raise RuntimeError("Test exception inside context")+ except RuntimeError:+ pass+ assert ProvenanceManager._default_storage_path is None++ def test_isolation_part_1_with_context_manager(self, tmp_path):+ """Part 1: Prove context manager sets default storage path for ProvenanceManager created inside context."""+ from semantica.provenance import default_storage_path, SQLiteStorage+ iso_db = str(tmp_path / "iso_part1.db")+ with default_storage_path(iso_db):+ prov_mgr = ProvenanceManager()+ assert isinstance(prov_mgr.storage, SQLiteStorage)+ assert prov_mgr.storage.db_path == iso_db++ def test_isolation_part_2_no_leakage_after_context(self):+ """Part 2: Prove that in a SEPARATE test function after context exit, ProvenanceManager falls back to InMemoryStorage without leakage."""+ from semantica.provenance import InMemoryStorage+ assert ProvenanceManager._default_storage_path is None+ prov_mgr = ProvenanceManager()+ assert isinstance(prov_mgr.storage, InMemoryStorage)++ def test_orchestrator_config_wiring(self, tmp_path):+ """Test Semantica orchestrator configures ProvenanceManager._default_storage_path."""+ from semantica.core import Semantica+ from semantica.provenance import ProvenanceManager, InMemoryStorage, SQLiteStorage++ # 1. No path set in config means InMemoryStorage fallback+ _ = Semantica()+ prov_mgr = ProvenanceManager()+ assert isinstance(prov_mgr.storage, InMemoryStorage)++ # 2. Config with provenance.storage_path sets default storage path+ test_db = str(tmp_path / "orch_test.db")+ _ = Semantica(config={"provenance": {"storage_path": test_db}})+ assert ProvenanceManager._default_storage_path == test_db+ prov_mgr = ProvenanceManager()+ assert isinstance(prov_mgr.storage, SQLiteStorage)+def test_initialization(self):"""Test manager initialization."""prov_mgr = ProvenanceManager()@@ -407,3 +489,42 @@ class TestProvenanceManager:assert e2.used_entities[0] in entity_ids, ("Archived history entry should be reachable via used_entities in lineage")++ def test_cli_lineage(self):+ """Test CLI lineage wrapper method on ProvenanceManager."""+ prov_mgr = ProvenanceManager()+ prov_mgr.track_entity("e_cli", source="doc_cli")+ res = prov_mgr.lineage("e_cli", depth=2)+ assert res["entity_id"] == "e_cli"+ assert res["depth"] == 2+ assert isinstance(res["lineage"], list)+ assert isinstance(res["sources"], list)++ def test_cli_audit_log(self):+ """Test CLI audit_log wrapper method on ProvenanceManager."""+ prov_mgr = ProvenanceManager()+ prov_mgr.track_entity("e_audit", source="doc_audit")+ res_table = prov_mgr.audit_log(format="table")+ assert isinstance(res_table, str)+ assert "ENTITY_ID" in res_table+ res_csv = prov_mgr.audit_log(format="csv")+ assert "entity_id,entity_type,activity_id,agent_id,timestamp" in res_csv+ res_json = prov_mgr.audit_log(format="json")+ assert isinstance(res_json, list)++ def test_cli_export_prov(self):+ """Test CLI export_prov method on ProvenanceManager."""+ prov_mgr = ProvenanceManager()+ prov_mgr.track_entity("e_rdf", source="doc_rdf")+ ttl = prov_mgr.export_prov(format="turtle")+ assert "prov:Entity" in ttl or "http://www.w3.org/ns/prov#Entity" in ttl++ def test_cli_check(self):+ """Test CLI check integrity method on ProvenanceManager."""+ prov_mgr = ProvenanceManager()+ prov_mgr.track_entity("e_valid", source="doc_1")+ check_res = prov_mgr.check(strict=True)+ assert check_res["valid"] is True+ assert check_res["total_entries"] >= 1+ assert check_res["errors"] == 0+