Skip to content

Personal AI, On Personal Devices

Python

feat: add SyncEngine with checkpoint/resume for connector orchestration

krypticmouseaf7d0db51828
diff --git a/src/openjarvis/connectors/sync_engine.py b/src/openjarvis/connectors/sync_engine.py
new file mode 100644
index 00000000..fe0eb501
--- /dev/null
+++ b/src/openjarvis/connectors/sync_engine.py
@@ -0,0 +1,183 @@
+"""SyncEngine — checkpoint/resume orchestration for connector syncs.
+
+Wraps ``IngestionPipeline`` with a lightweight SQLite state database so that
+long-running syncs can be interrupted and resumed from the last saved cursor.
+
+Typical usage::
+
+ store = KnowledgeStore(db_path=":memory:")
+ pipeline = IngestionPipeline(store)
+ engine = SyncEngine(pipeline)
+
+ items = engine.sync(connector) # first run
+ items = engine.sync(connector) # resumes from saved cursor
+ cp = engine.get_checkpoint(connector.connector_id)
+"""
+
+from __future__ import annotations
+
+import sqlite3
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from openjarvis.connectors._stubs import BaseConnector
+from openjarvis.connectors.pipeline import IngestionPipeline
+from openjarvis.core.config import DEFAULT_CONFIG_DIR
+
+# ---------------------------------------------------------------------------
+# DDL
+# ---------------------------------------------------------------------------
+
+_CREATE_STATE_TABLE = """
+CREATE TABLE IF NOT EXISTS sync_state (
+ connector_id TEXT PRIMARY KEY,
+ items_synced INTEGER NOT NULL DEFAULT 0,
+ cursor TEXT,
+ last_sync TEXT,
+ error TEXT
+);
+"""
+
+_BATCH_SIZE = 100
+
+
+class SyncEngine:
+ """Orchestrate connector syncs with checkpoint/resume tracking.
+
+ Parameters
+ ----------
+ pipeline:
+ The ``IngestionPipeline`` that documents are fed into.
+ state_db:
+ Path to the SQLite database used for checkpoint state. If empty,
+ defaults to ``DEFAULT_CONFIG_DIR / "sync_state.db"``.
+ """
+
+ def __init__(self, pipeline: IngestionPipeline, *, state_db: str = "") -> None:
+ self._pipeline = pipeline
+
+ if not state_db:
+ db_path = DEFAULT_CONFIG_DIR / "sync_state.db"
+ else:
+ db_path = Path(state_db)
+
+ # Ensure parent directory exists (skip for :memory:)
+ if str(db_path) != ":memory:":
+ db_path.parent.mkdir(parents=True, exist_ok=True)
+
+ self._conn = sqlite3.connect(str(db_path), check_same_thread=False)
+ self._conn.row_factory = sqlite3.Row
+ self._conn.execute("PRAGMA journal_mode=WAL;")
+ self._conn.execute(_CREATE_STATE_TABLE)
+ self._conn.commit()
+
+ # ------------------------------------------------------------------
+ # Public API
+ # ------------------------------------------------------------------
+
+ def sync(self, connector: BaseConnector) -> int:
+ """Run a full sync for *connector* and return the number of items ingested.
+
+ Resumes from the last saved cursor if one exists. Documents are
+ batched in groups of 100 before being handed to the pipeline; a
+ checkpoint is saved after every batch and once more at the end.
+
+ On error the checkpoint is updated with the error message and the
+ exception is re-raised so callers can handle it.
+ """
+ connector_id: str = connector.connector_id
+
+ # Load any previous checkpoint so we can resume.
+ checkpoint = self.get_checkpoint(connector_id)
+ prior_cursor: Optional[str] = checkpoint["cursor"] if checkpoint else None
+ prior_items: int = checkpoint["items_synced"] if checkpoint else 0
+
+ items_ingested = 0
+ current_cursor: Optional[str] = prior_cursor
+
+ try:
+ doc_iter = connector.sync(cursor=prior_cursor)
+
+ batch = []
+ for doc in doc_iter:
+ batch.append(doc)
+
+ if len(batch) >= _BATCH_SIZE:
+ items_ingested += self._pipeline.ingest(batch)
+ batch = []
+ self._save_checkpoint(
+ connector_id,
+ prior_items + items_ingested,
+ cursor=current_cursor,
+ )
+
+ # Ingest any remaining documents.
+ if batch:
+ items_ingested += self._pipeline.ingest(batch)
+
+ except Exception as exc:
+ self._save_checkpoint(
+ connector_id,
+ prior_items + items_ingested,
+ cursor=current_cursor,
+ error=str(exc),
+ )
+ raise
+
+ # Final checkpoint — clear any previous error.
+ self._save_checkpoint(
+ connector_id,
+ prior_items + items_ingested,
+ cursor=current_cursor,
+ error=None,
+ )
+ return items_ingested
+
+ def get_checkpoint(self, connector_id: str) -> Optional[Dict[str, Any]]:
+ """Return the last checkpoint for *connector_id*, or ``None`` if never synced."""
+ row = self._conn.execute(
+ "SELECT items_synced, cursor, last_sync, error FROM sync_state WHERE connector_id = ?",
+ (connector_id,),
+ ).fetchone()
+
+ if row is None:
+ return None
+
+ return {
+ "items_synced": row["items_synced"],
+ "cursor": row["cursor"],
+ "last_sync": row["last_sync"],
+ "error": row["error"],
+ }
+
+ # ------------------------------------------------------------------
+ # Internal helpers
+ # ------------------------------------------------------------------
+
+ def _save_checkpoint(
+ self,
+ connector_id: str,
+ items_synced: int,
+ *,
+ cursor: Optional[str] = None,
+ error: Optional[str] = None,
+ ) -> None:
+ """UPSERT a checkpoint row into the state database."""
+ now = datetime.now(tz=timezone.utc).isoformat()
+ self._conn.execute(
+ """
+ INSERT INTO sync_state (connector_id, items_synced, cursor, last_sync, error)
+ VALUES (?, ?, ?, ?, ?)
+ ON CONFLICT(connector_id) DO UPDATE SET
+ items_synced = excluded.items_synced,
+ cursor = excluded.cursor,
+ last_sync = excluded.last_sync,
+ error = excluded.error
+ """,
+ (connector_id, items_synced, cursor, now, error),
+ )
+ self._conn.commit()
+
+
+__all__ = ["SyncEngine"]
diff --git a/tests/connectors/test_sync_engine.py b/tests/connectors/test_sync_engine.py
new file mode 100644
index 00000000..2d202968
--- /dev/null
+++ b/tests/connectors/test_sync_engine.py
@@ -0,0 +1,174 @@
+"""Tests for SyncEngine — checkpoint/resume connector orchestration."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Iterator, List, Optional
+
+import pytest
+
+from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus
+from openjarvis.connectors.pipeline import IngestionPipeline
+from openjarvis.connectors.store import KnowledgeStore
+from openjarvis.connectors.sync_engine import SyncEngine
+
+
+# ---------------------------------------------------------------------------
+# StubConnector test helper
+# ---------------------------------------------------------------------------
+
+
+class StubConnector(BaseConnector):
+ """Minimal connector that replays a pre-built list of documents."""
+
+ connector_id = "stub"
+ display_name = "Stub"
+ auth_type = "filesystem"
+
+ def __init__(self, docs: List[Document]) -> None:
+ self._docs = docs
+
+ # BaseConnector abstract methods
+
+ def is_connected(self) -> bool:
+ return True
+
+ def disconnect(self) -> None:
+ pass
+
+ def sync(
+ self,
+ *,
+ since: Optional[datetime] = None,
+ cursor: Optional[str] = None,
+ ) -> Iterator[Document]:
+ yield from self._docs
+
+ def sync_status(self) -> SyncStatus:
+ return SyncStatus(state="idle", items_synced=len(self._docs))
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _make_doc(doc_id: str, source: str = "stub", content: str = "Test content.") -> Document:
+ return Document(
+ doc_id=doc_id,
+ source=source,
+ doc_type="note",
+ content=content,
+ title=f"Doc {doc_id}",
+ author="[email protected]",
+ timestamp=datetime(2025, 3, 1, tzinfo=timezone.utc),
+ )
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+def store(tmp_path: Path) -> KnowledgeStore:
+ return KnowledgeStore(db_path=tmp_path / "knowledge.db")
+
+
+def pipeline(store: KnowledgeStore) -> IngestionPipeline:
+ return IngestionPipeline(store)
+
+
+def engine(pipeline: IngestionPipeline, tmp_path: Path) -> SyncEngine:
+ return SyncEngine(pipeline, state_db=str(tmp_path / "sync_state.db"))
+
+
+# ---------------------------------------------------------------------------
+# Test 1: sync_connector — 5 docs from StubConnector all stored
+# ---------------------------------------------------------------------------
+
+
+def test_sync_connector(engine: SyncEngine, store: KnowledgeStore) -> None:
+ """StubConnector yields 5 docs; all are ingested and retrievable."""
+ docs = [_make_doc(f"doc:{i}", content=f"Unique content for document {i}") for i in range(5)]
+ connector = StubConnector(docs)
+
+ items = engine.sync(connector)
+
+ assert items == 5
+ # Each short doc produces exactly one chunk
+ assert store.count() == 5
+
+
+# ---------------------------------------------------------------------------
+# Test 2: sync_saves_checkpoint — checkpoint items_synced is correct
+# ---------------------------------------------------------------------------
+
+
+def test_sync_saves_checkpoint(engine: SyncEngine) -> None:
+ """After a sync, get_checkpoint returns the correct items_synced count."""
+ docs = [_make_doc(f"cp:doc:{i}") for i in range(3)]
+ connector = StubConnector(docs)
+
+ engine.sync(connector)
+
+ cp = engine.get_checkpoint("stub")
+ assert cp is not None
+ assert cp["items_synced"] == 3
+ assert cp["last_sync"] is not None
+ assert cp["error"] is None
+
+
+# ---------------------------------------------------------------------------
+# Test 3: sync_status_for_unsynced — None for unknown connector
+# ---------------------------------------------------------------------------
+
+
+def test_sync_status_for_unsynced(engine: SyncEngine) -> None:
+ """get_checkpoint returns None for a connector that has never been synced."""
+ result = engine.get_checkpoint("never_synced_connector")
+ assert result is None
+
+
+# ---------------------------------------------------------------------------
+# Test 4: sync_multiple_connectors — filter by source works
+# ---------------------------------------------------------------------------
+
+
+def test_sync_multiple_connectors(
+ pipeline: IngestionPipeline, store: KnowledgeStore, tmp_path: Path
+) -> None:
+ """Two connectors with different sources can be filtered independently in the store."""
+
+ class StubConnectorA(StubConnector):
+ connector_id = "stub_a"
+
+ class StubConnectorB(StubConnector):
+ connector_id = "stub_b"
+
+ docs_a = [_make_doc(f"a:doc:{i}", source="source_a", content=f"Alpha content {i}") for i in range(3)]
+ docs_b = [_make_doc(f"b:doc:{i}", source="source_b", content=f"Beta content {i}") for i in range(4)]
+
+ engine = SyncEngine(pipeline, state_db=str(tmp_path / "multi_sync_state.db"))
+
+ items_a = engine.sync(StubConnectorA(docs_a))
+ items_b = engine.sync(StubConnectorB(docs_b))
+
+ assert items_a == 3
+ assert items_b == 4
+ assert store.count() == 7
+
+ # Filter by source: only source_a results
+ results_a = store.retrieve("Alpha content", top_k=10, source="source_a")
+ assert len(results_a) >= 1
+ for r in results_a:
+ assert r.metadata.get("source") == "source_a"
+
+ # Filter by source: only source_b results
+ results_b = store.retrieve("Beta content", top_k=10, source="source_b")
+ assert len(results_b) >= 1
+ for r in results_b:
+ assert r.metadata.get("source") == "source_b"