Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(memory/graph): skip a corrupt row instead of aborting a whole graph scan (#3239)
Abhay Singh6262c28a4834
diff --git a/headroom/memory/adapters/sqlite_graph.py b/headroom/memory/adapters/sqlite_graph.pyindex 2c2ac432c..8c74bde6f 100644--- a/headroom/memory/adapters/sqlite_graph.py+++ b/headroom/memory/adapters/sqlite_graph.py@@ -14,6 +14,7 @@ This is a drop-in replacement for InMemoryGraphStore that:from __future__ import annotationsimport json+import loggingimport sqlite3from collections import dequefrom datetime import datetime@@ -26,6 +27,8 @@ from .graph_models import Entity, Relationship, RelationshipDirection, Subgraphif TYPE_CHECKING:from ..tracker import ComponentStats+logger = logging.getLogger(__name__)+class SQLiteGraphStore:"""SQLite-based graph store implementing the GraphStore protocol.@@ -165,19 +168,31 @@ class SQLiteGraphStore:"metadata": json.dumps(entity.metadata),}- def _row_to_entity(self, row: sqlite3.Row) -> Entity:- """Convert database row to Entity object."""- return Entity(- id=row["id"],- user_id=row["user_id"],- name=row["name"],- entity_type=row["entity_type"],- description=row["description"],- properties=json.loads(row["properties"]),- created_at=datetime.fromisoformat(row["created_at"]),- updated_at=datetime.fromisoformat(row["updated_at"]),- metadata=json.loads(row["metadata"]),- )+ def _row_to_entity(self, row: sqlite3.Row) -> Entity | None:+ """Convert a database row to an Entity, or None if the row is corrupt.++ ``properties``/``metadata`` (JSON) and ``created_at``/``updated_at``+ (ISO timestamps) are parsed from stored text. A single unparseable row —+ from a partial write, a manual edit, or a bad migration — must not abort+ an entire multi-row scan (``query_subgraph``, neighbour expansion): one+ corrupt edge would otherwise make an unrelated part of the graph+ unqueryable. Skip the bad row instead.+ """+ try:+ return Entity(+ id=row["id"],+ user_id=row["user_id"],+ name=row["name"],+ entity_type=row["entity_type"],+ description=row["description"],+ properties=json.loads(row["properties"]),+ created_at=datetime.fromisoformat(row["created_at"]),+ updated_at=datetime.fromisoformat(row["updated_at"]),+ metadata=json.loads(row["metadata"]),+ )+ except (ValueError, TypeError, KeyError) as exc:+ logger.warning("skipping corrupt entity row %r: %s", row["id"], exc)+ return Nonedef _relationship_to_row(self, relationship: Relationship) -> dict[str, Any]:"""Convert Relationship object to row dict for insertion."""@@ -193,19 +208,29 @@ class SQLiteGraphStore:"metadata": json.dumps(relationship.metadata),}- def _row_to_relationship(self, row: sqlite3.Row) -> Relationship:- """Convert database row to Relationship object."""- return Relationship(- id=row["id"],- user_id=row["user_id"],- source_id=row["source_id"],- target_id=row["target_id"],- relation_type=row["relation_type"],- weight=row["weight"],- properties=json.loads(row["properties"]),- created_at=datetime.fromisoformat(row["created_at"]),- metadata=json.loads(row["metadata"]),- )+ def _row_to_relationship(self, row: sqlite3.Row) -> Relationship | None:+ """Convert a database row to a Relationship, or None if the row is corrupt.++ Same contract as :meth:`_row_to_entity`: a single unparseable relationship+ row (bad ``properties``/``metadata`` JSON or ``created_at`` timestamp) must+ not abort a whole ``get_relationships`` / ``query_subgraph`` scan and take+ unrelated edges down with it. Skip the bad row instead.+ """+ try:+ return Relationship(+ id=row["id"],+ user_id=row["user_id"],+ source_id=row["source_id"],+ target_id=row["target_id"],+ relation_type=row["relation_type"],+ weight=row["weight"],+ properties=json.loads(row["properties"]),+ created_at=datetime.fromisoformat(row["created_at"]),+ metadata=json.loads(row["metadata"]),+ )+ except (ValueError, TypeError, KeyError) as exc:+ logger.warning("skipping corrupt relationship row %r: %s", row["id"], exc)+ return None# =========================================================================# Entity Operations@@ -373,7 +398,9 @@ class SQLiteGraphStore:params,)- return [self._row_to_relationship(row) for row in cursor]+ return [+ rel for row in cursor if (rel := self._row_to_relationship(row)) is not None+ ]async def delete_relationship(self, relationship_id: str) -> bool:"""Delete a single relationship.@@ -436,9 +463,12 @@ class SQLiteGraphStore:)row = cursor.fetchone()if row is not None:+ entity = self._row_to_entity(row)+ if entity is None:+ continuequeue.append((entity_id, 0))visited.add(entity_id)- collected_entities[entity_id] = self._row_to_entity(row)+ collected_entities[entity_id] = entity# BFS traversalwhile queue:@@ -470,6 +500,8 @@ class SQLiteGraphStore:for rel_row in cursor:rel = self._row_to_relationship(rel_row)+ if rel is None:+ continue# Add relationshipcollected_relationships[rel.id] = rel@@ -496,8 +528,11 @@ class SQLiteGraphStore:)neighbor_row = neighbor_cursor.fetchone()if neighbor_row is not None:+ neighbor = self._row_to_entity(neighbor_row)+ if neighbor is None:+ continuevisited.add(neighbor_id)- collected_entities[neighbor_id] = self._row_to_entity(neighbor_row)+ collected_entities[neighbor_id] = neighborqueue.append((neighbor_id, depth + 1))return Subgraph(@@ -651,7 +686,9 @@ class SQLiteGraphStore:"SELECT * FROM entities WHERE user_id = ?",(user_id,),)- return [self._row_to_entity(row) for row in cursor]+ return [+ entity for row in cursor if (entity := self._row_to_entity(row)) is not None+ ]async def clear(self) -> None:"""Clear all data from the store."""diff --git a/tests/test_sqlite_graph_store.py b/tests/test_sqlite_graph_store.pyindex 9fb5a90b0..b3db37e62 100644--- a/tests/test_sqlite_graph_store.py+++ b/tests/test_sqlite_graph_store.py@@ -689,6 +689,49 @@ class TestSQLiteGraphStoreEdgeCases:assert len(subgraph.entities) == 0assert len(subgraph.relationships) == 0+ @pytest.mark.asyncio+ async def test_one_corrupt_row_does_not_abort_a_multi_row_scan(self, tmp_path):+ """A single unparseable row must not take down an entire query.++ Regression: ``_row_to_relationship`` / ``_row_to_entity`` parsed stored+ JSON/timestamps with no guard, so one corrupt row (partial write, manual+ edit, bad migration) raised inside the row loop and aborted the whole+ ``query_subgraph`` / ``get_relationships`` scan — taking unrelated,+ perfectly good edges down with it. The corrupt row is now skipped.+ """+ import sqlite3++ store = SQLiteGraphStore(db_path=str(tmp_path / "graph.db"))+ a = Entity(user_id="u", name="A", entity_type="n")+ b = Entity(user_id="u", name="B", entity_type="n")+ c = Entity(user_id="u", name="C", entity_type="n")+ for entity in (a, b, c):+ await store.add_entity(entity)+ await store.add_relationship(+ Relationship(user_id="u", source_id=a.id, target_id=b.id, relation_type="e")+ )+ await store.add_relationship(+ Relationship(user_id="u", source_id=a.id, target_id=c.id, relation_type="e")+ )++ # Corrupt the A->B relationship row's properties JSON out-of-band.+ con = sqlite3.connect(str(store.db_path))+ con.execute("UPDATE relationships SET properties = '{oops' WHERE target_id = ?", (b.id,))+ con.commit()+ con.close()++ # get_relationships returns the one good edge instead of raising.+ rels = await store.get_relationships(a.id)+ assert len(rels) == 1+ assert rels[0].target_id == c.id++ # query_subgraph completes, skipping the corrupt edge and its node.+ subgraph = await store.query_subgraph(+ [a.id], max_hops=1, direction=RelationshipDirection.OUTGOING+ )+ assert {e.name for e in subgraph.entities} == {"A", "C"}+ assert len(subgraph.relationships) == 1+@pytest.mark.asyncioasync def test_entity_with_special_characters(self, store):"""Test entity names with special characters."""