Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
fix(provenance): address code review findings on PR #827
KaifAhmad10a8330cbb04e
diff --git a/semantica/explorer/routes/provenance.py b/semantica/explorer/routes/provenance.pyindex dd981862..daa52842 100644--- a/semantica/explorer/routes/provenance.py+++ b/semantica/explorer/routes/provenance.py@@ -82,7 +82,12 @@ def _add_chain_edges(seen_edges.add(edge_key)edges.append({- "id": f"{src}-{eid}",+ # Includes direction to match the seen_edges uniqueness key+ # above: the same (src, eid) pair can legitimately appear in+ # both directions (e.g. cycles/overlap between the ancestor+ # and descendant chains), and without this the two edges+ # would collide on the same id.+ "id": f"{src}-{eid}-{direction}","source": src,"target": eid,"label": activity,diff --git a/semantica/parse/parse_provenance.py b/semantica/parse/parse_provenance.pyindex 6d18386b..85a514ba 100644--- a/semantica/parse/parse_provenance.py+++ b/semantica/parse/parse_provenance.py@@ -13,7 +13,7 @@ Author: Semantica ContributorsLicense: MIT"""-from typing import Any, Optional+from typing import Optionalfrom datetime import datetimeimport uuiddiff --git a/semantica/provenance/manager.py b/semantica/provenance/manager.pyindex 8e411a17..8c2d1738 100644--- a/semantica/provenance/manager.py+++ b/semantica/provenance/manager.py@@ -1418,12 +1418,22 @@ class ProvenanceManager:Part A item 2).Sorts entries by sequence_id (global insertion order) and checks- both that each entry's own checksum matches its content, and that- each entry's previous_checksum matches the checksum of the entry- that precedes it. A gap in the chain — a surviving entry whose- previous_checksum doesn't match its predecessor's checksum — is- exactly what wholesale row deletion produces, since per-row- checksums alone can't detect that a row is simply missing.+ three things: each entry's own checksum matches its content; each+ entry's previous_checksum matches the checksum of the entry that+ precedes it; and sequence_id is exactly the predecessor's plus one+ (no gap, no duplicate). Every _save_entry() call assigns+ head_sequence + 1, and archival relabels (see track_entity's+ versioning and invalidate()) always preserve their existing+ sequence_id rather than consuming a new one — so under this design,+ the full set of currently-existing sequence_id values is always+ exactly {1..N} with nothing missing, unless a row was hard-deleted.+ A surviving entry whose previous_checksum doesn't match its+ predecessor, or whose sequence_id isn't predecessor+1, is exactly+ what wholesale row deletion produces; checking both signals also+ guards against the narrow case where two distinct rows happen to+ share a checksum (compute_checksum() deliberately excludes entity_id,+ see integrity.py), which alone would let a checksum-only check miss+ a gap that the sequence check still catches.Returns:Dictionary with "valid", "total_entries", and "broken_links"@@ -1436,6 +1446,7 @@ class ProvenanceManager:broken_links: List[Dict[str, Any]] = []expected_previous: Optional[str] = None+ expected_sequence: Optional[int] = Nonefor entry in entries:if not verify_checksum(entry):broken_links.append({@@ -1443,16 +1454,29 @@ class ProvenanceManager:"sequence_id": entry.sequence_id,"reason": "checksum_mismatch",})- continue- if entry.previous_checksum != expected_previous:- broken_links.append({- "entity_id": entry.entity_id,- "sequence_id": entry.sequence_id,- "reason": "chain_break",- "expected_previous_checksum": expected_previous,- "actual_previous_checksum": entry.previous_checksum,- })+ else:+ sequence_gap = (+ expected_sequence is not None+ and entry.sequence_id != expected_sequence + 1+ )+ checksum_break = entry.previous_checksum != expected_previous+ if sequence_gap or checksum_break:+ broken_links.append({+ "entity_id": entry.entity_id,+ "sequence_id": entry.sequence_id,+ "reason": "chain_break",+ "expected_previous_checksum": expected_previous,+ "actual_previous_checksum": entry.previous_checksum,+ "expected_sequence_id": (+ expected_sequence + 1 if expected_sequence is not None else None+ ),+ })++ # Advance state from this entry's own stored fields regardless of+ # whether it was flagged above, so a single corrupted entry+ # doesn't cascade into spurious breaks for every entry after it.expected_previous = entry.checksum+ expected_sequence = entry.sequence_idreturn {"valid": len(broken_links) == 0,diff --git a/semantica/provenance/storage.py b/semantica/provenance/storage.pyindex 768bc263..159fec5b 100644--- a/semantica/provenance/storage.py+++ b/semantica/provenance/storage.py@@ -532,13 +532,54 @@ class SQLiteStorage(ProvenanceStorage):finally:conn.close()+ # Columns added after the table was first shipped (issue #825, Part A/B).+ # CREATE TABLE IF NOT EXISTS alone does not alter an existing table, so a+ # provenance.db created before these columns existed would otherwise fail+ # on every insert/select once the row width and _row_to_entry's fixed+ # indices grew past the old schema. _migrate_schema() adds any of these+ # that are missing from an already-existing table.+ _MIGRATION_COLUMNS: List[Tuple[str, str]] = [+ ("agent_type", "TEXT DEFAULT 'software_agent'"),+ ("is_automated", "INTEGER DEFAULT 1"),+ ("role", "TEXT"),+ ("sequence_id", "INTEGER"),+ ("previous_checksum", "TEXT"),+ ("previous_version_id", "TEXT"),+ ("derived_from_id", "TEXT"),+ ("invalidated", "INTEGER DEFAULT 0"),+ ("invalidated_at_time", "TEXT"),+ ("invalidated_by", "TEXT"),+ ("invalidation_reason", "TEXT"),+ ("activity_started_at_time", "TEXT"),+ ("activity_ended_at_time", "TEXT"),+ ("acted_on_behalf_of", "TEXT"),+ ("informed_by_activities", "TEXT"),+ ("valid_from", "TEXT"),+ ("valid_until", "TEXT"),+ ("revision_type", "TEXT"),+ ("supersedes", "TEXT"),+ ("bundle_id", "TEXT"),+ ]++ def _migrate_schema(self, conn: sqlite3.Connection) -> None:+ """Add any columns introduced after the table was first created to an+ already-existing table (see _MIGRATION_COLUMNS)."""+ cursor = conn.cursor()+ cursor.execute("PRAGMA table_info(provenance)")+ existing_columns = {row[1] for row in cursor.fetchall()}+ for column_name, column_def in self._MIGRATION_COLUMNS:+ if column_name not in existing_columns:+ cursor.execute(+ f"ALTER TABLE provenance ADD COLUMN {column_name} {column_def}"+ )+def _init_db(self) -> None:"""Create tables with W3C PROV-O compliant schema."""conn = sqlite3.connect(self.db_path)try:self._configure_connection(conn)cursor = conn.cursor()-+cursor.execute("""CREATE TABLE IF NOT EXISTS provenance (entity_id TEXT PRIMARY KEY,@@ -583,6 +624,11 @@ class SQLiteStorage(ProvenanceStorage):)""")+ # Add any columns missing from a table that already existed+ # before these were introduced (see _MIGRATION_COLUMNS) — a no-op+ # for a table that was just freshly created above.+ self._migrate_schema(conn)+# Create indexes for efficient queryingcursor.execute("""CREATE INDEX IF NOT EXISTS idx_entity_typediff --git a/tests/explorer/test_provenance_route.py b/tests/explorer/test_provenance_route.pyindex 73f59bf2..aa2b9d72 100644--- a/tests/explorer/test_provenance_route.py+++ b/tests/explorer/test_provenance_route.py@@ -2,7 +2,11 @@from types import SimpleNamespace-from semantica.explorer.routes.provenance import _build_provenance, _render_markdown+from semantica.explorer.routes.provenance import (+ _add_chain_edges,+ _build_provenance,+ _render_markdown,+)def _make_session_with_chain() -> SimpleNamespace:@@ -72,3 +76,25 @@ def test_render_markdown_groups_edges_by_direction():assert "## Lateral" in markdownassert "`Intermediate` -[related_to]-> `node_id`" in markdownassert "`Source` -[related_to]-> `Intermediate`" in markdown+++def test_add_chain_edges_ids_distinguish_direction():+ """Regression test: the same (src, target) pair appearing in both the+ ancestor and descendant chains (e.g. a cycle or overlap between them)+ must not collide on edge id — the id must carry the same uniqueness+ information as the seen_edges dedupe key, which already includes+ direction."""+ same_pair_chain = [+ {"entity_id": "b", "parent_entity_id": "a", "activity_id": "act"},+ ]++ edges: list = []+ seen_edges: set = set()+ _add_chain_edges(same_pair_chain, edges, seen_edges, "upstream")+ _add_chain_edges(same_pair_chain, edges, seen_edges, "downstream")++ assert len(edges) == 2+ ids = {edge["id"] for edge in edges}+ assert len(ids) == 2, f"edge ids collided across directions: {edges}"+ directions = {edge["id"]: edge["direction"] for edge in edges}+ assert directions == {"a-b-upstream": "upstream", "a-b-downstream": "downstream"}diff --git a/tests/provenance/test_manager.py b/tests/provenance/test_manager.pyindex e8c7f44a..e9f14a6c 100644--- a/tests/provenance/test_manager.py+++ b/tests/provenance/test_manager.py@@ -1102,6 +1102,31 @@ class TestHashChain:assert result["valid"] is Falseassert any(link["reason"] == "chain_break" for link in result["broken_links"])+ def test_verify_chain_detects_tampered_sequence_gap(self, tmp_path):+ """The sequence_id continuity check catches a gap introduced by+ directly tampering with a row's sequence_id column, independent of+ the previous_checksum comparison — hardening against the narrow case+ where compute_checksum()'s deliberate exclusion of entity_id could+ otherwise let two distinct rows coincidentally share a checksum."""+ import sqlite3+ db_path = str(tmp_path / "chain_seq_gap.db")+ prov_mgr = ProvenanceManager(storage_path=db_path)+ prov_mgr.track_entity("e1", source="doc1")+ prov_mgr.track_entity("e2", source="doc1")+ prov_mgr.track_entity("e3", source="doc1")++ conn = sqlite3.connect(db_path)+ conn.execute("UPDATE provenance SET sequence_id = 10 WHERE entity_id = 'e3'")+ conn.commit()+ conn.close()++ prov_mgr2 = ProvenanceManager(storage_path=db_path)+ result = prov_mgr2.verify_chain()+ assert result["valid"] is False+ broken = [link for link in result["broken_links"] if link["entity_id"] == "e3"]+ assert broken+ assert broken[0]["expected_sequence_id"] == 3+def test_sequence_ids_are_assigned_and_monotonic(self):prov_mgr = ProvenanceManager()e1 = prov_mgr.track_entity("e1", source="doc1")diff --git a/tests/provenance/test_storage.py b/tests/provenance/test_storage.pyindex 5c35764e..673fbb8f 100644--- a/tests/provenance/test_storage.py+++ b/tests/provenance/test_storage.py@@ -5,6 +5,7 @@ Tests for InMemoryStorage and SQLiteStorage backends."""import pytest+import sqlite3import tempfileimport osfrom semantica.provenance.schemas import ProvenanceEntry@@ -194,6 +195,79 @@ class TestInMemoryStorage:class TestSQLiteStorage:"""Test SQLiteStorage backend."""+ def test_migrates_pre_existing_old_schema_database(self):+ """Regression test: SQLiteStorage._init_db() previously only ran+ CREATE TABLE IF NOT EXISTS, which does not alter an existing table.+ A provenance.db created before issue #825's Part A/B columns existed+ would otherwise break on every insert/select once the new code's+ wider positional INSERT and _row_to_entry's fixed indices no longer+ matched the old (narrower) row width. SQLiteStorage now migrates any+ missing columns in on open."""+ with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as tmp:+ db_path = tmp.name++ try:+ # Simulate a database created by the pre-#825 code: only the+ # original 19 columns, no agent_type/sequence_id/valid_from/etc.+ conn = sqlite3.connect(db_path)+ conn.execute("""+ CREATE TABLE provenance (+ entity_id TEXT PRIMARY KEY,+ entity_type TEXT NOT NULL,+ activity_id TEXT NOT NULL,+ agent_id TEXT DEFAULT 'semantica',+ source_document TEXT,+ source_location TEXT,+ source_quote TEXT,+ timestamp TEXT NOT NULL,+ first_seen TEXT,+ last_updated TEXT,+ confidence REAL DEFAULT 1.0,+ checksum TEXT,+ parent_entity_id TEXT,+ used_entities TEXT,+ start_index INTEGER,+ end_index INTEGER,+ credibility REAL,+ metadata TEXT,+ version TEXT DEFAULT '1.0'+ )+ """)+ conn.execute(+ "INSERT INTO provenance VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",+ (+ "old_entity", "entity", "legacy_activity", "semantica", "doc1",+ None, None, "2025-01-01T00:00:00", "2025-01-01T00:00:00",+ "2025-01-01T00:00:00", 1.0, "oldchecksum", None, "[]",+ None, None, None, "{}", "1.0",+ ),+ )+ conn.commit()+ conn.close()++ # Opening with the current SQLiteStorage must not crash, and must+ # migrate the table in place rather than requiring a fresh file.+ storage = SQLiteStorage(db_path)++ old_entry = storage.retrieve("old_entity")+ assert old_entry is not None+ assert old_entry.entity_id == "old_entity"+ assert old_entry.agent_type == "software_agent" # new column default+ assert old_entry.sequence_id is None # never assigned pre-migration++ # New writes against the migrated table must work too.+ new_entry = ProvenanceEntry(+ entity_id="new_entity", entity_type="entity", activity_id="act",+ agent_id="alice", sequence_id=1, checksum="chk1",+ )+ storage.store(new_entry)+ retrieved = storage.retrieve("new_entity")+ assert retrieved.agent_id == "alice"+ assert retrieved.sequence_id == 1+ finally:+ if os.path.exists(db_path):+ os.unlink(db_path)+def test_all_fields_round_trip_through_sqlite(self):"""Regression test: every ProvenanceEntry field, including issue #825Part A and Part B additions, must survive a SQLite store/retrieve