Personal AI, On Personal Devices
Python
fix(connectors): don't advance last_sync on failed/in-progress checkpoints (#782) (#806)
havolline1a5374ae2749b
diff --git a/src/openjarvis/connectors/sync_engine.py b/src/openjarvis/connectors/sync_engine.pyindex a9c43363..a2333e17 100644--- a/src/openjarvis/connectors/sync_engine.py+++ b/src/openjarvis/connectors/sync_engine.py@@ -100,6 +100,11 @@ class SyncEngine:except (ValueError, TypeError):pass+ # Captured before the first fetch, not at completion, so items+ # created/modified while this sync was running are still covered+ # by the *next* sync's `since` filter instead of being skipped.+ sync_started_at = datetime.now(tz=timezone.utc).isoformat()+items_ingested = 0current_cursor: Optional[str] = prior_cursor@@ -113,6 +118,12 @@ class SyncEngine:if len(batch) >= _BATCH_SIZE:items_ingested += self._pipeline.ingest(batch)batch = []+ # Progress checkpoint: track cursor/items so a later+ # retry can resume from here, but must NOT advance+ # last_sync -- this sync hasn't completed yet, and+ # advancing the watermark on a checkpoint that isn't a+ # successful completion would permanently skip+ # whatever wasn't reached if the sync then fails (#782).self._save_checkpoint(connector_id,prior_items + items_ingested,@@ -132,12 +143,14 @@ class SyncEngine:)raise- # Final checkpoint — clear any previous error.+ # Final checkpoint on successful completion — clear any previous+ # error and advance the watermark to when this sync started.self._save_checkpoint(connector_id,prior_items + items_ingested,cursor=current_cursor,error=None,+ last_sync=sync_started_at,)return items_ingested@@ -170,9 +183,16 @@ class SyncEngine:*,cursor: Optional[str] = None,error: Optional[str] = None,+ last_sync: Optional[str] = None,) -> None:- """UPSERT a checkpoint row into the state database."""- now = datetime.now(tz=timezone.utc).isoformat()+ """UPSERT a checkpoint row into the state database.++ ``last_sync`` only advances when the caller explicitly passes a+ value (a successful sync completion). ``COALESCE`` preserves the+ existing watermark on progress/error checkpoints, so a failed or+ still-in-progress sync can never permanently skip unprocessed+ items (#782).+ """self._conn.execute("""INSERT INTO sync_state@@ -181,10 +201,10 @@ class SyncEngine:ON CONFLICT(connector_id) DO UPDATE SETitems_synced = excluded.items_synced,cursor = excluded.cursor,- last_sync = excluded.last_sync,+ last_sync = COALESCE(excluded.last_sync, sync_state.last_sync),error = excluded.error""",- (connector_id, items_synced, cursor, now, error),+ (connector_id, items_synced, cursor, last_sync, error),)self._conn.commit()diff --git a/tests/connectors/test_sync_engine.py b/tests/connectors/test_sync_engine.pyindex 5b82709d..8d7d1e9d 100644--- a/tests/connectors/test_sync_engine.py+++ b/tests/connectors/test_sync_engine.py@@ -48,6 +48,46 @@ class StubConnector(BaseConnector):return SyncStatus(state="idle", items_synced=len(self._docs))+class RecordingConnector(BaseConnector):+ """Replays documents, recording `since` on each call, optionally+ raising partway through to simulate a sync that dies mid-stream."""++ connector_id = "recorder"+ display_name = "Recorder"+ auth_type = "filesystem"++ def __init__(+ self,+ docs: List[Document],+ *,+ fail_after: Optional[int] = None,+ ) -> None:+ self._docs = docs+ self._fail_after = fail_after+ self.received_since: List[Optional[datetime]] = []++ def is_connected(self) -> bool:+ return True++ def disconnect(self) -> None:+ pass++ def sync(+ self,+ *,+ since: Optional[datetime] = None,+ cursor: Optional[str] = None,+ ) -> Iterator[Document]:+ self.received_since.append(since)+ for i, doc in enumerate(self._docs):+ if self._fail_after is not None and i == self._fail_after:+ raise RuntimeError("connector exploded mid-sync")+ yield doc++ def sync_status(self) -> SyncStatus:+ return SyncStatus(state="idle", items_synced=len(self._docs))++# ---------------------------------------------------------------------------# Helpers# ---------------------------------------------------------------------------@@ -182,3 +222,52 @@ def test_sync_multiple_connectors(assert len(results_b) >= 1for r in results_b:assert r.metadata.get("source") == "source_b"+++# ---------------------------------------------------------------------------+# Test 5: a failed sync must not advance last_sync (#782)+# ---------------------------------------------------------------------------+++def test_failed_sync_does_not_advance_last_sync(engine: SyncEngine) -> None:+ """A sync that dies partway through must preserve the prior watermark+ (None here, since this is the connector's first-ever sync) instead of+ recording the failure time as last_sync -- otherwise a retry would+ start from the failure time and permanently skip whatever the failed+ attempt never reached."""+ docs = [_make_doc(f"fail:doc:{i}") for i in range(5)]+ connector = RecordingConnector(docs, fail_after=3)++ with pytest.raises(RuntimeError, match="connector exploded mid-sync"):+ engine.sync(connector)++ cp = engine.get_checkpoint("recorder")+ assert cp is not None+ assert cp["last_sync"] is None+ assert cp["error"] is not None+++def test_retry_after_failure_does_not_skip_the_failed_window(+ engine: SyncEngine,+) -> None:+ """End-to-end: after a failed sync, a retry must see the SAME `since`+ as the original attempt (None, since nothing ever succeeded) -- not a+ timestamp advanced by the failed attempt, which would silently skip+ the entire window the failure never got to process."""+ docs = [_make_doc(f"retry:doc:{i}") for i in range(5)]++ failing = RecordingConnector(docs, fail_after=3)+ with pytest.raises(RuntimeError):+ engine.sync(failing)+ assert failing.received_since == [None]++ retry = RecordingConnector(docs, fail_after=None)+ items = engine.sync(retry)++ assert retry.received_since == [None]+ assert items == 5++ cp = engine.get_checkpoint("recorder")+ assert cp is not None+ assert cp["last_sync"] is not None+ assert cp["error"] is None