Personal AI, On Personal Devices
Python
feat: wire incremental sync via since param in SyncEngine
krypticmouse25d6a66d68a8
diff --git a/src/openjarvis/connectors/sync_engine.py b/src/openjarvis/connectors/sync_engine.pyindex 111521c1..a9c43363 100644--- a/src/openjarvis/connectors/sync_engine.py+++ b/src/openjarvis/connectors/sync_engine.py@@ -93,11 +93,18 @@ class SyncEngine:prior_cursor: Optional[str] = checkpoint["cursor"] if checkpoint else Noneprior_items: int = checkpoint["items_synced"] if checkpoint else 0+ since: Optional[datetime] = None+ if checkpoint and checkpoint.get("last_sync"):+ try:+ since = datetime.fromisoformat(checkpoint["last_sync"])+ except (ValueError, TypeError):+ pass+items_ingested = 0current_cursor: Optional[str] = prior_cursortry:- doc_iter = connector.sync(cursor=prior_cursor)+ doc_iter = connector.sync(since=since, cursor=prior_cursor)batch = []for doc in doc_iter:diff --git a/tests/connectors/test_incremental_sync.py b/tests/connectors/test_incremental_sync.pynew file mode 100644index 00000000..20cda4ba--- /dev/null+++ b/tests/connectors/test_incremental_sync.py@@ -0,0 +1,186 @@+"""Tests for incremental sync via the `since` parameter in SyncEngine."""++from __future__ import annotations++from datetime import datetime, timezone+from pathlib import Path+from typing import Iterator, List, Optional++import pytest++from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus+from openjarvis.connectors.pipeline import IngestionPipeline+from openjarvis.connectors.store import KnowledgeStore+from openjarvis.connectors.sync_engine import SyncEngine++# ---------------------------------------------------------------------------+# TimestampConnector — records the `since` value it receives+# ---------------------------------------------------------------------------+++class TimestampConnector(BaseConnector):+ """Connector that records the `since` argument passed to each sync call."""++ connector_id = "timestamp_connector"+ display_name = "Timestamp"+ auth_type = "filesystem"++ def __init__(self, docs: List[Document]) -> None:+ self._docs = docs+ self.received_since: List[Optional[datetime]] = []++ # BaseConnector abstract methods++ def is_connected(self) -> bool:+ return True++ def disconnect(self) -> None:+ pass++ def sync(+ self,+ *,+ since: Optional[datetime] = None,+ cursor: Optional[str] = None,+ ) -> Iterator[Document]:+ self.received_since.append(since)+ yield from self._docs++ def sync_status(self) -> SyncStatus:+ return SyncStatus(state="idle", items_synced=len(self._docs))+++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------+++def _make_doc(doc_id: str, content: str = "Test content.") -> Document:+ return Document(+ doc_id=doc_id,+ source="timestamp_connector",+ doc_type="note",+ content=content,+ title=f"Doc {doc_id}",+ author="[email protected]",+ timestamp=datetime(2025, 3, 1, tzinfo=timezone.utc),+ )+++# ---------------------------------------------------------------------------+# Fixtures+# ---------------------------------------------------------------------------+++def store(tmp_path: Path) -> KnowledgeStore:+ return KnowledgeStore(db_path=tmp_path / "knowledge.db")+++def pipeline(store: KnowledgeStore) -> IngestionPipeline:+ return IngestionPipeline(store)+++def engine(pipeline: IngestionPipeline, tmp_path: Path) -> SyncEngine:+ return SyncEngine(pipeline, state_db=str(tmp_path / "sync_state.db"))+++# ---------------------------------------------------------------------------+# Test 1: first sync has no checkpoint — since should be None+# ---------------------------------------------------------------------------+++def test_first_sync_passes_no_since(engine: SyncEngine) -> None:+ """On the very first sync there is no checkpoint, so since must be None."""+ docs = [_make_doc("first:doc:0", content="First sync document")]+ connector = TimestampConnector(docs)++ engine.sync(connector)++ assert len(connector.received_since) == 1+ assert connector.received_since[0] is None+++# ---------------------------------------------------------------------------+# Test 2: second sync should receive a non-None since datetime+# ---------------------------------------------------------------------------+++def test_second_sync_passes_since(engine: SyncEngine) -> None:+ """After the first sync a checkpoint is written; the second sync receives since != None."""+ docs = [_make_doc("second:doc:0", content="Document for incremental sync")]+ connector = TimestampConnector(docs)++ # First sync — establishes the checkpoint+ engine.sync(connector)+ assert connector.received_since[0] is None++ # Second sync — should receive a datetime parsed from last_sync+ engine.sync(connector)+ assert len(connector.received_since) == 2+ since_value = connector.received_since[1]+ assert since_value is not None+ assert isinstance(since_value, datetime)+++# ---------------------------------------------------------------------------+# Test 3: incremental sync only adds newly returned items+# ---------------------------------------------------------------------------+++def test_incremental_only_adds_new_items(+ pipeline: IngestionPipeline, store: KnowledgeStore, tmp_path: Path+) -> None:+ """Connector returns only new docs on the second call; store grows by that count."""++ class SelectiveTimestampConnector(TimestampConnector):+ """Returns a different doc list on each sync call."""++ def __init__(+ self, first_docs: List[Document], second_docs: List[Document]+ ) -> None:+ super().__init__(first_docs)+ self._first_docs = first_docs+ self._second_docs = second_docs+ self._call_count = 0++ def sync(+ self,+ *,+ since: Optional[datetime] = None,+ cursor: Optional[str] = None,+ ) -> Iterator[Document]:+ self.received_since.append(since)+ self._call_count += 1+ if self._call_count == 1:+ yield from self._first_docs+ else:+ yield from self._second_docs++ old_docs = [+ _make_doc("incr:old:0", content="Old document alpha"),+ _make_doc("incr:old:1", content="Old document beta"),+ ]+ new_docs = [+ _make_doc("incr:new:0", content="New document gamma"),+ ]++ engine = SyncEngine(pipeline, state_db=str(tmp_path / "incr_sync_state.db"))+ connector = SelectiveTimestampConnector(old_docs, new_docs)++ # First sync — 2 old docs ingested+ items_first = engine.sync(connector)+ assert items_first == 2+ count_after_first = store.count()+ assert count_after_first == 2++ # Second sync — only 1 new doc returned by the connector+ items_second = engine.sync(connector)+ assert items_second == 1+ assert store.count() == 3++ # The second call received a valid since datetime+ assert connector.received_since[1] is not None+ assert isinstance(connector.received_since[1], datetime)