Personal AI, On Personal Devices
Python
feat: add SyncEngine with checkpoint/resume for connector orchestration
krypticmouseaf7d0db51828
diff --git a/src/openjarvis/connectors/sync_engine.py b/src/openjarvis/connectors/sync_engine.pynew file mode 100644index 00000000..fe0eb501--- /dev/null+++ b/src/openjarvis/connectors/sync_engine.py@@ -0,0 +1,183 @@+"""SyncEngine — checkpoint/resume orchestration for connector syncs.++Wraps ``IngestionPipeline`` with a lightweight SQLite state database so that+long-running syncs can be interrupted and resumed from the last saved cursor.++Typical usage::++ store = KnowledgeStore(db_path=":memory:")+ pipeline = IngestionPipeline(store)+ engine = SyncEngine(pipeline)++ items = engine.sync(connector) # first run+ items = engine.sync(connector) # resumes from saved cursor+ cp = engine.get_checkpoint(connector.connector_id)+"""++from __future__ import annotations++import sqlite3+from datetime import datetime, timezone+from pathlib import Path+from typing import Any, Dict, Optional++from openjarvis.connectors._stubs import BaseConnector+from openjarvis.connectors.pipeline import IngestionPipeline+from openjarvis.core.config import DEFAULT_CONFIG_DIR++# ---------------------------------------------------------------------------+# DDL+# ---------------------------------------------------------------------------++_CREATE_STATE_TABLE = """+CREATE TABLE IF NOT EXISTS sync_state (+ connector_id TEXT PRIMARY KEY,+ items_synced INTEGER NOT NULL DEFAULT 0,+ cursor TEXT,+ last_sync TEXT,+ error TEXT+);+"""++_BATCH_SIZE = 100+++class SyncEngine:+ """Orchestrate connector syncs with checkpoint/resume tracking.++ Parameters+ ----------+ pipeline:+ The ``IngestionPipeline`` that documents are fed into.+ state_db:+ Path to the SQLite database used for checkpoint state. If empty,+ defaults to ``DEFAULT_CONFIG_DIR / "sync_state.db"``.+ """++ def __init__(self, pipeline: IngestionPipeline, *, state_db: str = "") -> None:+ self._pipeline = pipeline++ if not state_db:+ db_path = DEFAULT_CONFIG_DIR / "sync_state.db"+ else:+ db_path = Path(state_db)++ # Ensure parent directory exists (skip for :memory:)+ if str(db_path) != ":memory:":+ db_path.parent.mkdir(parents=True, exist_ok=True)++ self._conn = sqlite3.connect(str(db_path), check_same_thread=False)+ self._conn.row_factory = sqlite3.Row+ self._conn.execute("PRAGMA journal_mode=WAL;")+ self._conn.execute(_CREATE_STATE_TABLE)+ self._conn.commit()++ # ------------------------------------------------------------------+ # Public API+ # ------------------------------------------------------------------++ def sync(self, connector: BaseConnector) -> int:+ """Run a full sync for *connector* and return the number of items ingested.++ Resumes from the last saved cursor if one exists. Documents are+ batched in groups of 100 before being handed to the pipeline; a+ checkpoint is saved after every batch and once more at the end.++ On error the checkpoint is updated with the error message and the+ exception is re-raised so callers can handle it.+ """+ connector_id: str = connector.connector_id++ # Load any previous checkpoint so we can resume.+ checkpoint = self.get_checkpoint(connector_id)+ prior_cursor: Optional[str] = checkpoint["cursor"] if checkpoint else None+ prior_items: int = checkpoint["items_synced"] if checkpoint else 0++ items_ingested = 0+ current_cursor: Optional[str] = prior_cursor++ try:+ doc_iter = connector.sync(cursor=prior_cursor)++ batch = []+ for doc in doc_iter:+ batch.append(doc)++ if len(batch) >= _BATCH_SIZE:+ items_ingested += self._pipeline.ingest(batch)+ batch = []+ self._save_checkpoint(+ connector_id,+ prior_items + items_ingested,+ cursor=current_cursor,+ )++ # Ingest any remaining documents.+ if batch:+ items_ingested += self._pipeline.ingest(batch)++ except Exception as exc:+ self._save_checkpoint(+ connector_id,+ prior_items + items_ingested,+ cursor=current_cursor,+ error=str(exc),+ )+ raise++ # Final checkpoint — clear any previous error.+ self._save_checkpoint(+ connector_id,+ prior_items + items_ingested,+ cursor=current_cursor,+ error=None,+ )+ return items_ingested++ def get_checkpoint(self, connector_id: str) -> Optional[Dict[str, Any]]:+ """Return the last checkpoint for *connector_id*, or ``None`` if never synced."""+ row = self._conn.execute(+ "SELECT items_synced, cursor, last_sync, error FROM sync_state WHERE connector_id = ?",+ (connector_id,),+ ).fetchone()++ if row is None:+ return None++ return {+ "items_synced": row["items_synced"],+ "cursor": row["cursor"],+ "last_sync": row["last_sync"],+ "error": row["error"],+ }++ # ------------------------------------------------------------------+ # Internal helpers+ # ------------------------------------------------------------------++ def _save_checkpoint(+ self,+ connector_id: str,+ items_synced: int,+ *,+ cursor: Optional[str] = None,+ error: Optional[str] = None,+ ) -> None:+ """UPSERT a checkpoint row into the state database."""+ now = datetime.now(tz=timezone.utc).isoformat()+ self._conn.execute(+ """+ INSERT INTO sync_state (connector_id, items_synced, cursor, last_sync, error)+ VALUES (?, ?, ?, ?, ?)+ ON CONFLICT(connector_id) DO UPDATE SET+ items_synced = excluded.items_synced,+ cursor = excluded.cursor,+ last_sync = excluded.last_sync,+ error = excluded.error+ """,+ (connector_id, items_synced, cursor, now, error),+ )+ self._conn.commit()+++__all__ = ["SyncEngine"]diff --git a/tests/connectors/test_sync_engine.py b/tests/connectors/test_sync_engine.pynew file mode 100644index 00000000..2d202968--- /dev/null+++ b/tests/connectors/test_sync_engine.py@@ -0,0 +1,174 @@+"""Tests for SyncEngine — checkpoint/resume connector orchestration."""++from __future__ import annotations++from datetime import datetime, timezone+from pathlib import Path+from typing import Iterator, List, Optional++import pytest++from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus+from openjarvis.connectors.pipeline import IngestionPipeline+from openjarvis.connectors.store import KnowledgeStore+from openjarvis.connectors.sync_engine import SyncEngine+++# ---------------------------------------------------------------------------+# StubConnector test helper+# ---------------------------------------------------------------------------+++class StubConnector(BaseConnector):+ """Minimal connector that replays a pre-built list of documents."""++ connector_id = "stub"+ display_name = "Stub"+ auth_type = "filesystem"++ def __init__(self, docs: List[Document]) -> None:+ self._docs = docs++ # BaseConnector abstract methods++ def is_connected(self) -> bool:+ return True++ def disconnect(self) -> None:+ pass++ def sync(+ self,+ *,+ since: Optional[datetime] = None,+ cursor: Optional[str] = None,+ ) -> Iterator[Document]:+ yield from self._docs++ def sync_status(self) -> SyncStatus:+ return SyncStatus(state="idle", items_synced=len(self._docs))+++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------+++def _make_doc(doc_id: str, source: str = "stub", content: str = "Test content.") -> Document:+ return Document(+ doc_id=doc_id,+ source=source,+ doc_type="note",+ content=content,+ title=f"Doc {doc_id}",+ author="[email protected]",+ timestamp=datetime(2025, 3, 1, tzinfo=timezone.utc),+ )+++# ---------------------------------------------------------------------------+# Fixtures+# ---------------------------------------------------------------------------+++def store(tmp_path: Path) -> KnowledgeStore:+ return KnowledgeStore(db_path=tmp_path / "knowledge.db")+++def pipeline(store: KnowledgeStore) -> IngestionPipeline:+ return IngestionPipeline(store)+++def engine(pipeline: IngestionPipeline, tmp_path: Path) -> SyncEngine:+ return SyncEngine(pipeline, state_db=str(tmp_path / "sync_state.db"))+++# ---------------------------------------------------------------------------+# Test 1: sync_connector — 5 docs from StubConnector all stored+# ---------------------------------------------------------------------------+++def test_sync_connector(engine: SyncEngine, store: KnowledgeStore) -> None:+ """StubConnector yields 5 docs; all are ingested and retrievable."""+ docs = [_make_doc(f"doc:{i}", content=f"Unique content for document {i}") for i in range(5)]+ connector = StubConnector(docs)++ items = engine.sync(connector)++ assert items == 5+ # Each short doc produces exactly one chunk+ assert store.count() == 5+++# ---------------------------------------------------------------------------+# Test 2: sync_saves_checkpoint — checkpoint items_synced is correct+# ---------------------------------------------------------------------------+++def test_sync_saves_checkpoint(engine: SyncEngine) -> None:+ """After a sync, get_checkpoint returns the correct items_synced count."""+ docs = [_make_doc(f"cp:doc:{i}") for i in range(3)]+ connector = StubConnector(docs)++ engine.sync(connector)++ cp = engine.get_checkpoint("stub")+ assert cp is not None+ assert cp["items_synced"] == 3+ assert cp["last_sync"] is not None+ assert cp["error"] is None+++# ---------------------------------------------------------------------------+# Test 3: sync_status_for_unsynced — None for unknown connector+# ---------------------------------------------------------------------------+++def test_sync_status_for_unsynced(engine: SyncEngine) -> None:+ """get_checkpoint returns None for a connector that has never been synced."""+ result = engine.get_checkpoint("never_synced_connector")+ assert result is None+++# ---------------------------------------------------------------------------+# Test 4: sync_multiple_connectors — filter by source works+# ---------------------------------------------------------------------------+++def test_sync_multiple_connectors(+ pipeline: IngestionPipeline, store: KnowledgeStore, tmp_path: Path+) -> None:+ """Two connectors with different sources can be filtered independently in the store."""++ class StubConnectorA(StubConnector):+ connector_id = "stub_a"++ class StubConnectorB(StubConnector):+ connector_id = "stub_b"++ docs_a = [_make_doc(f"a:doc:{i}", source="source_a", content=f"Alpha content {i}") for i in range(3)]+ docs_b = [_make_doc(f"b:doc:{i}", source="source_b", content=f"Beta content {i}") for i in range(4)]++ engine = SyncEngine(pipeline, state_db=str(tmp_path / "multi_sync_state.db"))++ items_a = engine.sync(StubConnectorA(docs_a))+ items_b = engine.sync(StubConnectorB(docs_b))++ assert items_a == 3+ assert items_b == 4+ assert store.count() == 7++ # Filter by source: only source_a results+ results_a = store.retrieve("Alpha content", top_k=10, source="source_a")+ assert len(results_a) >= 1+ for r in results_a:+ assert r.metadata.get("source") == "source_a"++ # Filter by source: only source_b results+ results_b = store.retrieve("Beta content", top_k=10, source="source_b")+ assert len(results_b) >= 1+ for r in results_b:+ assert r.metadata.get("source") == "source_b"