Skip to content

Personal AI, On Personal Devices

Python

fix(connectors): don't advance last_sync on failed/in-progress checkpoints (#782) (#806)

havolline1a5374ae2749b
diff --git a/src/openjarvis/connectors/sync_engine.py b/src/openjarvis/connectors/sync_engine.py
index a9c43363..a2333e17 100644
--- a/src/openjarvis/connectors/sync_engine.py
+++ b/src/openjarvis/connectors/sync_engine.py
@@ -100,6 +100,11 @@ class SyncEngine:
except (ValueError, TypeError):
pass
+ # Captured before the first fetch, not at completion, so items
+ # created/modified while this sync was running are still covered
+ # by the *next* sync's `since` filter instead of being skipped.
+ sync_started_at = datetime.now(tz=timezone.utc).isoformat()
+
items_ingested = 0
current_cursor: Optional[str] = prior_cursor
@@ -113,6 +118,12 @@ class SyncEngine:
if len(batch) >= _BATCH_SIZE:
items_ingested += self._pipeline.ingest(batch)
batch = []
+ # Progress checkpoint: track cursor/items so a later
+ # retry can resume from here, but must NOT advance
+ # last_sync -- this sync hasn't completed yet, and
+ # advancing the watermark on a checkpoint that isn't a
+ # successful completion would permanently skip
+ # whatever wasn't reached if the sync then fails (#782).
self._save_checkpoint(
connector_id,
prior_items + items_ingested,
@@ -132,12 +143,14 @@ class SyncEngine:
)
raise
- # Final checkpoint — clear any previous error.
+ # Final checkpoint on successful completion — clear any previous
+ # error and advance the watermark to when this sync started.
self._save_checkpoint(
connector_id,
prior_items + items_ingested,
cursor=current_cursor,
error=None,
+ last_sync=sync_started_at,
)
return items_ingested
@@ -170,9 +183,16 @@ class SyncEngine:
*,
cursor: Optional[str] = None,
error: Optional[str] = None,
+ last_sync: Optional[str] = None,
) -> None:
- """UPSERT a checkpoint row into the state database."""
- now = datetime.now(tz=timezone.utc).isoformat()
+ """UPSERT a checkpoint row into the state database.
+
+ ``last_sync`` only advances when the caller explicitly passes a
+ value (a successful sync completion). ``COALESCE`` preserves the
+ existing watermark on progress/error checkpoints, so a failed or
+ still-in-progress sync can never permanently skip unprocessed
+ items (#782).
+ """
self._conn.execute(
"""
INSERT INTO sync_state
@@ -181,10 +201,10 @@ class SyncEngine:
ON CONFLICT(connector_id) DO UPDATE SET
items_synced = excluded.items_synced,
cursor = excluded.cursor,
- last_sync = excluded.last_sync,
+ last_sync = COALESCE(excluded.last_sync, sync_state.last_sync),
error = excluded.error
""",
- (connector_id, items_synced, cursor, now, error),
+ (connector_id, items_synced, cursor, last_sync, error),
)
self._conn.commit()
diff --git a/tests/connectors/test_sync_engine.py b/tests/connectors/test_sync_engine.py
index 5b82709d..8d7d1e9d 100644
--- a/tests/connectors/test_sync_engine.py
+++ b/tests/connectors/test_sync_engine.py
@@ -48,6 +48,46 @@ class StubConnector(BaseConnector):
return SyncStatus(state="idle", items_synced=len(self._docs))
+class RecordingConnector(BaseConnector):
+ """Replays documents, recording `since` on each call, optionally
+ raising partway through to simulate a sync that dies mid-stream."""
+
+ connector_id = "recorder"
+ display_name = "Recorder"
+ auth_type = "filesystem"
+
+ def __init__(
+ self,
+ docs: List[Document],
+ *,
+ fail_after: Optional[int] = None,
+ ) -> None:
+ self._docs = docs
+ self._fail_after = fail_after
+ self.received_since: List[Optional[datetime]] = []
+
+ def is_connected(self) -> bool:
+ return True
+
+ def disconnect(self) -> None:
+ pass
+
+ def sync(
+ self,
+ *,
+ since: Optional[datetime] = None,
+ cursor: Optional[str] = None,
+ ) -> Iterator[Document]:
+ self.received_since.append(since)
+ for i, doc in enumerate(self._docs):
+ if self._fail_after is not None and i == self._fail_after:
+ raise RuntimeError("connector exploded mid-sync")
+ yield doc
+
+ def sync_status(self) -> SyncStatus:
+ return SyncStatus(state="idle", items_synced=len(self._docs))
+
+
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
@@ -182,3 +222,52 @@ def test_sync_multiple_connectors(
assert len(results_b) >= 1
for r in results_b:
assert r.metadata.get("source") == "source_b"
+
+
+# ---------------------------------------------------------------------------
+# Test 5: a failed sync must not advance last_sync (#782)
+# ---------------------------------------------------------------------------
+
+
+def test_failed_sync_does_not_advance_last_sync(engine: SyncEngine) -> None:
+ """A sync that dies partway through must preserve the prior watermark
+ (None here, since this is the connector's first-ever sync) instead of
+ recording the failure time as last_sync -- otherwise a retry would
+ start from the failure time and permanently skip whatever the failed
+ attempt never reached."""
+ docs = [_make_doc(f"fail:doc:{i}") for i in range(5)]
+ connector = RecordingConnector(docs, fail_after=3)
+
+ with pytest.raises(RuntimeError, match="connector exploded mid-sync"):
+ engine.sync(connector)
+
+ cp = engine.get_checkpoint("recorder")
+ assert cp is not None
+ assert cp["last_sync"] is None
+ assert cp["error"] is not None
+
+
+def test_retry_after_failure_does_not_skip_the_failed_window(
+ engine: SyncEngine,
+) -> None:
+ """End-to-end: after a failed sync, a retry must see the SAME `since`
+ as the original attempt (None, since nothing ever succeeded) -- not a
+ timestamp advanced by the failed attempt, which would silently skip
+ the entire window the failure never got to process."""
+ docs = [_make_doc(f"retry:doc:{i}") for i in range(5)]
+
+ failing = RecordingConnector(docs, fail_after=3)
+ with pytest.raises(RuntimeError):
+ engine.sync(failing)
+ assert failing.received_since == [None]
+
+ retry = RecordingConnector(docs, fail_after=None)
+ items = engine.sync(retry)
+
+ assert retry.received_since == [None]
+ assert items == 5
+
+ cp = engine.get_checkpoint("recorder")
+ assert cp is not None
+ assert cp["last_sync"] is not None
+ assert cp["error"] is None