Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

fix(provenance): address code review findings on PR #827

KaifAhmad10a8330cbb04e
diff --git a/semantica/explorer/routes/provenance.py b/semantica/explorer/routes/provenance.py
index dd981862..daa52842 100644
--- a/semantica/explorer/routes/provenance.py
+++ b/semantica/explorer/routes/provenance.py
@@ -82,7 +82,12 @@ def _add_chain_edges(
seen_edges.add(edge_key)
edges.append({
- "id": f"{src}-{eid}",
+ # Includes direction to match the seen_edges uniqueness key
+ # above: the same (src, eid) pair can legitimately appear in
+ # both directions (e.g. cycles/overlap between the ancestor
+ # and descendant chains), and without this the two edges
+ # would collide on the same id.
+ "id": f"{src}-{eid}-{direction}",
"source": src,
"target": eid,
"label": activity,
diff --git a/semantica/parse/parse_provenance.py b/semantica/parse/parse_provenance.py
index 6d18386b..85a514ba 100644
--- a/semantica/parse/parse_provenance.py
+++ b/semantica/parse/parse_provenance.py
@@ -13,7 +13,7 @@ Author: Semantica Contributors
License: MIT
"""
-from typing import Any, Optional
+from typing import Optional
from datetime import datetime
import uuid
diff --git a/semantica/provenance/manager.py b/semantica/provenance/manager.py
index 8e411a17..8c2d1738 100644
--- a/semantica/provenance/manager.py
+++ b/semantica/provenance/manager.py
@@ -1418,12 +1418,22 @@ class ProvenanceManager:
Part A item 2).
Sorts entries by sequence_id (global insertion order) and checks
- both that each entry's own checksum matches its content, and that
- each entry's previous_checksum matches the checksum of the entry
- that precedes it. A gap in the chain — a surviving entry whose
- previous_checksum doesn't match its predecessor's checksum — is
- exactly what wholesale row deletion produces, since per-row
- checksums alone can't detect that a row is simply missing.
+ three things: each entry's own checksum matches its content; each
+ entry's previous_checksum matches the checksum of the entry that
+ precedes it; and sequence_id is exactly the predecessor's plus one
+ (no gap, no duplicate). Every _save_entry() call assigns
+ head_sequence + 1, and archival relabels (see track_entity's
+ versioning and invalidate()) always preserve their existing
+ sequence_id rather than consuming a new one — so under this design,
+ the full set of currently-existing sequence_id values is always
+ exactly {1..N} with nothing missing, unless a row was hard-deleted.
+ A surviving entry whose previous_checksum doesn't match its
+ predecessor, or whose sequence_id isn't predecessor+1, is exactly
+ what wholesale row deletion produces; checking both signals also
+ guards against the narrow case where two distinct rows happen to
+ share a checksum (compute_checksum() deliberately excludes entity_id,
+ see integrity.py), which alone would let a checksum-only check miss
+ a gap that the sequence check still catches.
Returns:
Dictionary with "valid", "total_entries", and "broken_links"
@@ -1436,6 +1446,7 @@ class ProvenanceManager:
broken_links: List[Dict[str, Any]] = []
expected_previous: Optional[str] = None
+ expected_sequence: Optional[int] = None
for entry in entries:
if not verify_checksum(entry):
broken_links.append({
@@ -1443,16 +1454,29 @@ class ProvenanceManager:
"sequence_id": entry.sequence_id,
"reason": "checksum_mismatch",
})
- continue
- if entry.previous_checksum != expected_previous:
- broken_links.append({
- "entity_id": entry.entity_id,
- "sequence_id": entry.sequence_id,
- "reason": "chain_break",
- "expected_previous_checksum": expected_previous,
- "actual_previous_checksum": entry.previous_checksum,
- })
+ else:
+ sequence_gap = (
+ expected_sequence is not None
+ and entry.sequence_id != expected_sequence + 1
+ )
+ checksum_break = entry.previous_checksum != expected_previous
+ if sequence_gap or checksum_break:
+ broken_links.append({
+ "entity_id": entry.entity_id,
+ "sequence_id": entry.sequence_id,
+ "reason": "chain_break",
+ "expected_previous_checksum": expected_previous,
+ "actual_previous_checksum": entry.previous_checksum,
+ "expected_sequence_id": (
+ expected_sequence + 1 if expected_sequence is not None else None
+ ),
+ })
+
+ # Advance state from this entry's own stored fields regardless of
+ # whether it was flagged above, so a single corrupted entry
+ # doesn't cascade into spurious breaks for every entry after it.
expected_previous = entry.checksum
+ expected_sequence = entry.sequence_id
return {
"valid": len(broken_links) == 0,
diff --git a/semantica/provenance/storage.py b/semantica/provenance/storage.py
index 768bc263..159fec5b 100644
--- a/semantica/provenance/storage.py
+++ b/semantica/provenance/storage.py
@@ -532,13 +532,54 @@ class SQLiteStorage(ProvenanceStorage):
finally:
conn.close()
+ # Columns added after the table was first shipped (issue #825, Part A/B).
+ # CREATE TABLE IF NOT EXISTS alone does not alter an existing table, so a
+ # provenance.db created before these columns existed would otherwise fail
+ # on every insert/select once the row width and _row_to_entry's fixed
+ # indices grew past the old schema. _migrate_schema() adds any of these
+ # that are missing from an already-existing table.
+ _MIGRATION_COLUMNS: List[Tuple[str, str]] = [
+ ("agent_type", "TEXT DEFAULT 'software_agent'"),
+ ("is_automated", "INTEGER DEFAULT 1"),
+ ("role", "TEXT"),
+ ("sequence_id", "INTEGER"),
+ ("previous_checksum", "TEXT"),
+ ("previous_version_id", "TEXT"),
+ ("derived_from_id", "TEXT"),
+ ("invalidated", "INTEGER DEFAULT 0"),
+ ("invalidated_at_time", "TEXT"),
+ ("invalidated_by", "TEXT"),
+ ("invalidation_reason", "TEXT"),
+ ("activity_started_at_time", "TEXT"),
+ ("activity_ended_at_time", "TEXT"),
+ ("acted_on_behalf_of", "TEXT"),
+ ("informed_by_activities", "TEXT"),
+ ("valid_from", "TEXT"),
+ ("valid_until", "TEXT"),
+ ("revision_type", "TEXT"),
+ ("supersedes", "TEXT"),
+ ("bundle_id", "TEXT"),
+ ]
+
+ def _migrate_schema(self, conn: sqlite3.Connection) -> None:
+ """Add any columns introduced after the table was first created to an
+ already-existing table (see _MIGRATION_COLUMNS)."""
+ cursor = conn.cursor()
+ cursor.execute("PRAGMA table_info(provenance)")
+ existing_columns = {row[1] for row in cursor.fetchall()}
+ for column_name, column_def in self._MIGRATION_COLUMNS:
+ if column_name not in existing_columns:
+ cursor.execute(
+ f"ALTER TABLE provenance ADD COLUMN {column_name} {column_def}"
+ )
+
def _init_db(self) -> None:
"""Create tables with W3C PROV-O compliant schema."""
conn = sqlite3.connect(self.db_path)
try:
self._configure_connection(conn)
cursor = conn.cursor()
-
+
cursor.execute("""
CREATE TABLE IF NOT EXISTS provenance (
entity_id TEXT PRIMARY KEY,
@@ -583,6 +624,11 @@ class SQLiteStorage(ProvenanceStorage):
)
""")
+ # Add any columns missing from a table that already existed
+ # before these were introduced (see _MIGRATION_COLUMNS) — a no-op
+ # for a table that was just freshly created above.
+ self._migrate_schema(conn)
+
# Create indexes for efficient querying
cursor.execute("""
CREATE INDEX IF NOT EXISTS idx_entity_type
diff --git a/tests/explorer/test_provenance_route.py b/tests/explorer/test_provenance_route.py
index 73f59bf2..aa2b9d72 100644
--- a/tests/explorer/test_provenance_route.py
+++ b/tests/explorer/test_provenance_route.py
@@ -2,7 +2,11 @@
from types import SimpleNamespace
-from semantica.explorer.routes.provenance import _build_provenance, _render_markdown
+from semantica.explorer.routes.provenance import (
+ _add_chain_edges,
+ _build_provenance,
+ _render_markdown,
+)
def _make_session_with_chain() -> SimpleNamespace:
@@ -72,3 +76,25 @@ def test_render_markdown_groups_edges_by_direction():
assert "## Lateral" in markdown
assert "`Intermediate` -[related_to]-> `node_id`" in markdown
assert "`Source` -[related_to]-> `Intermediate`" in markdown
+
+
+def test_add_chain_edges_ids_distinguish_direction():
+ """Regression test: the same (src, target) pair appearing in both the
+ ancestor and descendant chains (e.g. a cycle or overlap between them)
+ must not collide on edge id — the id must carry the same uniqueness
+ information as the seen_edges dedupe key, which already includes
+ direction."""
+ same_pair_chain = [
+ {"entity_id": "b", "parent_entity_id": "a", "activity_id": "act"},
+ ]
+
+ edges: list = []
+ seen_edges: set = set()
+ _add_chain_edges(same_pair_chain, edges, seen_edges, "upstream")
+ _add_chain_edges(same_pair_chain, edges, seen_edges, "downstream")
+
+ assert len(edges) == 2
+ ids = {edge["id"] for edge in edges}
+ assert len(ids) == 2, f"edge ids collided across directions: {edges}"
+ directions = {edge["id"]: edge["direction"] for edge in edges}
+ assert directions == {"a-b-upstream": "upstream", "a-b-downstream": "downstream"}
diff --git a/tests/provenance/test_manager.py b/tests/provenance/test_manager.py
index e8c7f44a..e9f14a6c 100644
--- a/tests/provenance/test_manager.py
+++ b/tests/provenance/test_manager.py
@@ -1102,6 +1102,31 @@ class TestHashChain:
assert result["valid"] is False
assert any(link["reason"] == "chain_break" for link in result["broken_links"])
+ def test_verify_chain_detects_tampered_sequence_gap(self, tmp_path):
+ """The sequence_id continuity check catches a gap introduced by
+ directly tampering with a row's sequence_id column, independent of
+ the previous_checksum comparison — hardening against the narrow case
+ where compute_checksum()'s deliberate exclusion of entity_id could
+ otherwise let two distinct rows coincidentally share a checksum."""
+ import sqlite3
+ db_path = str(tmp_path / "chain_seq_gap.db")
+ prov_mgr = ProvenanceManager(storage_path=db_path)
+ prov_mgr.track_entity("e1", source="doc1")
+ prov_mgr.track_entity("e2", source="doc1")
+ prov_mgr.track_entity("e3", source="doc1")
+
+ conn = sqlite3.connect(db_path)
+ conn.execute("UPDATE provenance SET sequence_id = 10 WHERE entity_id = 'e3'")
+ conn.commit()
+ conn.close()
+
+ prov_mgr2 = ProvenanceManager(storage_path=db_path)
+ result = prov_mgr2.verify_chain()
+ assert result["valid"] is False
+ broken = [link for link in result["broken_links"] if link["entity_id"] == "e3"]
+ assert broken
+ assert broken[0]["expected_sequence_id"] == 3
+
def test_sequence_ids_are_assigned_and_monotonic(self):
prov_mgr = ProvenanceManager()
e1 = prov_mgr.track_entity("e1", source="doc1")
diff --git a/tests/provenance/test_storage.py b/tests/provenance/test_storage.py
index 5c35764e..673fbb8f 100644
--- a/tests/provenance/test_storage.py
+++ b/tests/provenance/test_storage.py
@@ -5,6 +5,7 @@ Tests for InMemoryStorage and SQLiteStorage backends.
"""
import pytest
+import sqlite3
import tempfile
import os
from semantica.provenance.schemas import ProvenanceEntry
@@ -194,6 +195,79 @@ class TestInMemoryStorage:
class TestSQLiteStorage:
"""Test SQLiteStorage backend."""
+ def test_migrates_pre_existing_old_schema_database(self):
+ """Regression test: SQLiteStorage._init_db() previously only ran
+ CREATE TABLE IF NOT EXISTS, which does not alter an existing table.
+ A provenance.db created before issue #825's Part A/B columns existed
+ would otherwise break on every insert/select once the new code's
+ wider positional INSERT and _row_to_entry's fixed indices no longer
+ matched the old (narrower) row width. SQLiteStorage now migrates any
+ missing columns in on open."""
+ with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as tmp:
+ db_path = tmp.name
+
+ try:
+ # Simulate a database created by the pre-#825 code: only the
+ # original 19 columns, no agent_type/sequence_id/valid_from/etc.
+ conn = sqlite3.connect(db_path)
+ conn.execute("""
+ CREATE TABLE provenance (
+ entity_id TEXT PRIMARY KEY,
+ entity_type TEXT NOT NULL,
+ activity_id TEXT NOT NULL,
+ agent_id TEXT DEFAULT 'semantica',
+ source_document TEXT,
+ source_location TEXT,
+ source_quote TEXT,
+ timestamp TEXT NOT NULL,
+ first_seen TEXT,
+ last_updated TEXT,
+ confidence REAL DEFAULT 1.0,
+ checksum TEXT,
+ parent_entity_id TEXT,
+ used_entities TEXT,
+ start_index INTEGER,
+ end_index INTEGER,
+ credibility REAL,
+ metadata TEXT,
+ version TEXT DEFAULT '1.0'
+ )
+ """)
+ conn.execute(
+ "INSERT INTO provenance VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+ (
+ "old_entity", "entity", "legacy_activity", "semantica", "doc1",
+ None, None, "2025-01-01T00:00:00", "2025-01-01T00:00:00",
+ "2025-01-01T00:00:00", 1.0, "oldchecksum", None, "[]",
+ None, None, None, "{}", "1.0",
+ ),
+ )
+ conn.commit()
+ conn.close()
+
+ # Opening with the current SQLiteStorage must not crash, and must
+ # migrate the table in place rather than requiring a fresh file.
+ storage = SQLiteStorage(db_path)
+
+ old_entry = storage.retrieve("old_entity")
+ assert old_entry is not None
+ assert old_entry.entity_id == "old_entity"
+ assert old_entry.agent_type == "software_agent" # new column default
+ assert old_entry.sequence_id is None # never assigned pre-migration
+
+ # New writes against the migrated table must work too.
+ new_entry = ProvenanceEntry(
+ entity_id="new_entity", entity_type="entity", activity_id="act",
+ agent_id="alice", sequence_id=1, checksum="chk1",
+ )
+ storage.store(new_entry)
+ retrieved = storage.retrieve("new_entity")
+ assert retrieved.agent_id == "alice"
+ assert retrieved.sequence_id == 1
+ finally:
+ if os.path.exists(db_path):
+ os.unlink(db_path)
+
def test_all_fields_round_trip_through_sqlite(self):
"""Regression test: every ProvenanceEntry field, including issue #825
Part A and Part B additions, must survive a SQLite store/retrieve