Skip to content

Personal AI, On Personal Devices

Python

fix: reset connector sync checkpoint and purge indexed content on disconnect (#819)

havolline1dd6a9ddb8d77
diff --git a/src/openjarvis/connectors/_stubs.py b/src/openjarvis/connectors/_stubs.py
index e956bfe8..a8023845 100644
--- a/src/openjarvis/connectors/_stubs.py
+++ b/src/openjarvis/connectors/_stubs.py
@@ -74,6 +74,11 @@ class BaseConnector(ABC):
connector_id: str
display_name: str
auth_type: str # "oauth" | "local" | "bridge" | "filesystem"
+ indexed_sources: tuple[str, ...] = ()
+
+ def knowledge_sources(self) -> tuple[str, ...]:
+ """Return KnowledgeStore source values owned by this connector."""
+ return self.indexed_sources or (self.connector_id,)
@abstractmethod
def is_connected(self) -> bool:
diff --git a/src/openjarvis/connectors/gmail_imap.py b/src/openjarvis/connectors/gmail_imap.py
index 1d00d2a2..107d4a3f 100644
--- a/src/openjarvis/connectors/gmail_imap.py
+++ b/src/openjarvis/connectors/gmail_imap.py
@@ -145,6 +145,7 @@ class GmailIMAPConnector(BaseConnector):
connector_id = "gmail_imap"
display_name = "Gmail (IMAP)"
auth_type = "oauth" # Reuses credential storage pattern
+ indexed_sources = ("gmail",)
_default_imap_host = "imap.gmail.com"
def __init__(
diff --git a/src/openjarvis/connectors/store.py b/src/openjarvis/connectors/store.py
index 7fff7d51..3e6300d2 100644
--- a/src/openjarvis/connectors/store.py
+++ b/src/openjarvis/connectors/store.py
@@ -15,7 +15,7 @@ import time
import uuid
from datetime import datetime, timezone
from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, Iterable, List, Optional, Union
from openjarvis.core.events import EventType, get_event_bus
from openjarvis.core.registry import MemoryRegistry
@@ -454,6 +454,37 @@ class KnowledgeStore(MemoryBackend):
self._conn.commit()
return cur.rowcount > 0
+ def delete_by_source(self, source: str) -> int:
+ """Delete all chunks with the given *source*. Returns the count removed.
+
+ Must be called when a connector is disconnected so a later
+ reconnect -- to the same or a different underlying data source --
+ starts clean. Without this, orphaned chunks from the old source
+ remain indexed forever, and if the new source produces a doc_id
+ that happens to collide with one of them (e.g. two vaults each
+ containing a file at the same relative path), the ingestion
+ pipeline's duplicate-doc_id dedup silently discards the new
+ content in favor of the stale orphan.
+ """
+ return self.delete_by_sources((source,))
+
+ def delete_by_sources(self, sources: Iterable[str]) -> int:
+ """Atomically delete chunks belonging to any of *sources*."""
+ unique_sources = tuple(dict.fromkeys(sources))
+ if not unique_sources:
+ return 0
+ placeholders = ", ".join("?" for _ in unique_sources)
+ try:
+ cur = self._conn.execute(
+ f"DELETE FROM knowledge_chunks WHERE source IN ({placeholders})",
+ unique_sources,
+ )
+ self._conn.commit()
+ except Exception:
+ self._conn.rollback()
+ raise
+ return cur.rowcount
+
def clear(self) -> None:
"""Remove all stored chunks."""
self._conn.executescript(
diff --git a/src/openjarvis/connectors/sync_engine.py b/src/openjarvis/connectors/sync_engine.py
index a2333e17..58908b1b 100644
--- a/src/openjarvis/connectors/sync_engine.py
+++ b/src/openjarvis/connectors/sync_engine.py
@@ -17,6 +17,7 @@ Typical usage::
from __future__ import annotations
import sqlite3
+import threading
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, Optional
@@ -72,11 +73,22 @@ class SyncEngine:
self._conn.execute(_CREATE_STATE_TABLE)
self._conn.commit()
+ def __enter__(self) -> "SyncEngine":
+ return self
+
+ def __exit__(self, *exc_info: object) -> None:
+ self.close()
+
# ------------------------------------------------------------------
# Public API
# ------------------------------------------------------------------
- def sync(self, connector: BaseConnector) -> int:
+ def sync(
+ self,
+ connector: BaseConnector,
+ *,
+ cancel_event: Optional[threading.Event] = None,
+ ) -> int:
"""Run a full sync for *connector* and return the number of items ingested.
Resumes from the last saved cursor if one exists. Documents are
@@ -113,6 +125,8 @@ class SyncEngine:
batch = []
for doc in doc_iter:
+ if cancel_event is not None and cancel_event.is_set():
+ break
batch.append(doc)
if len(batch) >= _BATCH_SIZE:
@@ -131,7 +145,7 @@ class SyncEngine:
)
# Ingest any remaining documents.
- if batch:
+ if batch and not (cancel_event is not None and cancel_event.is_set()):
items_ingested += self._pipeline.ingest(batch)
except Exception as exc:
@@ -143,6 +157,19 @@ class SyncEngine:
)
raise
+ if cancel_event is not None and cancel_event.is_set():
+ # Cancellation means the source was not exhausted. Preserve the
+ # prior successful watermark exactly as failure checkpoints do;
+ # advancing it here could make the next sync skip documents that
+ # the cancelled run never reached (#782).
+ self._save_checkpoint(
+ connector_id,
+ prior_items + items_ingested,
+ cursor=current_cursor,
+ error=None,
+ )
+ return items_ingested
+
# Final checkpoint on successful completion — clear any previous
# error and advance the watermark to when this sync started.
self._save_checkpoint(
@@ -154,6 +181,10 @@ class SyncEngine:
)
return items_ingested
+ def close(self) -> None:
+ """Close the checkpoint database connection."""
+ self._conn.close()
+
def get_checkpoint(self, connector_id: str) -> Optional[Dict[str, Any]]:
"""Return the last checkpoint, or ``None`` if never synced."""
sql = (
@@ -172,6 +203,51 @@ class SyncEngine:
"error": row["error"],
}
+ def reset_checkpoint(self, connector_id: str) -> None:
+ """Clear the saved checkpoint for *connector_id*.
+
+ Must be called whenever a connector is reconfigured to point at a
+ different underlying data source (e.g. disconnect/reconnect with a
+ new Obsidian vault path). Without this, the next sync would resume
+ from the old source's cursor/``since`` watermark, inflating
+ ``items_synced`` with the old source's count and potentially
+ skipping new items whose timestamps predate that watermark.
+ """
+ self._conn.execute(
+ "DELETE FROM sync_state WHERE connector_id = ?", (connector_id,)
+ )
+ self._conn.commit()
+
+ def restore_checkpoint(
+ self,
+ connector_id: str,
+ checkpoint: Optional[Dict[str, Any]],
+ ) -> None:
+ """Restore an exact checkpoint snapshot after a failed cleanup."""
+ if checkpoint is None:
+ self.reset_checkpoint(connector_id)
+ return
+ self._conn.execute(
+ """
+ INSERT INTO sync_state
+ (connector_id, items_synced, cursor, last_sync, error)
+ VALUES (?, ?, ?, ?, ?)
+ ON CONFLICT(connector_id) DO UPDATE SET
+ items_synced = excluded.items_synced,
+ cursor = excluded.cursor,
+ last_sync = excluded.last_sync,
+ error = excluded.error
+ """,
+ (
+ connector_id,
+ checkpoint["items_synced"],
+ checkpoint.get("cursor"),
+ checkpoint.get("last_sync"),
+ checkpoint.get("error"),
+ ),
+ )
+ self._conn.commit()
+
# ------------------------------------------------------------------
# Internal helpers
# ------------------------------------------------------------------
diff --git a/src/openjarvis/server/connectors_router.py b/src/openjarvis/server/connectors_router.py
index 367865b1..18e39b7b 100644
--- a/src/openjarvis/server/connectors_router.py
+++ b/src/openjarvis/server/connectors_router.py
@@ -3,6 +3,8 @@
from __future__ import annotations
import logging
+import threading
+from functools import wraps
from typing import TYPE_CHECKING, Any, Dict, Optional
# ``Request`` must be importable at *module* scope so that FastAPI can resolve
@@ -25,6 +27,17 @@ logger = logging.getLogger(__name__)
# Module-level cache of connector instances (keyed by connector_id).
_instances: Dict[str, Any] = {}
+_SYNC_STOP_TIMEOUT_SECONDS = 5.0
+
+
+def _knowledge_sources(connector_id: str, connector: Any = None) -> tuple[str, ...]:
+ """Return the KnowledgeStore source values owned by a connector."""
+ if connector is not None and not isinstance(connector, type):
+ getter = getattr(connector, "knowledge_sources", None)
+ if callable(getter):
+ return tuple(getter())
+ explicit = getattr(connector, "indexed_sources", ())
+ return tuple(explicit) if explicit else (connector_id,)
def _ensure_connectors_registered() -> None:
@@ -127,10 +140,13 @@ def create_connectors_router():
try:
from openjarvis.connectors.store import KnowledgeStore
+ sources = _knowledge_sources(connector_id, instance)
+ placeholders = ", ".join("?" for _ in sources)
with KnowledgeStore() as store:
rows = store._conn.execute(
- "SELECT COUNT(*) FROM knowledge_chunks WHERE source = ?",
- (connector_id,),
+ f"SELECT COUNT(*) FROM knowledge_chunks "
+ f"WHERE source IN ({placeholders})",
+ sources,
).fetchone()
chunks = rows[0] if rows else 0
except Exception:
@@ -215,7 +231,64 @@ def create_connectors_router():
# ------------------------------------------------------------------
_sync_threads: Dict[str, Any] = {}
+ _sync_cancel_events: Dict[str, threading.Event] = {}
_sync_state: Dict[str, Dict[str, Any]] = {}
+ _sync_lock = threading.Lock()
+ _sync_generations: Dict[str, int] = {}
+ # Connector lifecycle mutations are rare and touch shared credential,
+ # instance, checkpoint, and source-ownership state. Serializing them
+ # globally keeps connect/disconnect/manual-sync decisions atomic,
+ # including ownership checks for sources shared by multiple connectors.
+ _lifecycle_lock = threading.RLock()
+
+ def _disconnect_pending(connector_id: str) -> bool:
+ """Whether a prior disconnect is waiting for its worker to stop."""
+ with _sync_lock:
+ cancel_event = _sync_cancel_events.get(connector_id)
+ return (
+ connector_id in _sync_threads
+ and cancel_event is not None
+ and cancel_event.is_set()
+ )
+
+ def _reject_pending_disconnect(connector_id: str) -> None:
+ """Prevent credential/source changes while an old worker still owns them."""
+ if _disconnect_pending(connector_id):
+ raise HTTPException(
+ status_code=409,
+ detail=(
+ f"Sync for '{connector_id}' is still stopping; "
+ "retry disconnect before reconnecting or syncing"
+ ),
+ )
+
+ def _serialized_async(func):
+ @wraps(func)
+ async def wrapped(*args, **kwargs):
+ with _lifecycle_lock:
+ return await func(*args, **kwargs)
+
+ return wrapped
+
+ def _serialized_sync(func):
+ @wraps(func)
+ def wrapped(*args, **kwargs):
+ with _lifecycle_lock:
+ return func(*args, **kwargs)
+
+ return wrapped
+
+ def _publish_sync_state(
+ connector_id: str,
+ generation: int,
+ state: Dict[str, Any],
+ ) -> bool:
+ """Publish worker state only while it still owns this lifecycle."""
+ with _sync_lock:
+ if _sync_generations.get(connector_id, 0) != generation:
+ return False
+ _sync_state[connector_id] = state
+ return True
def _translate_sync_error(raw: str) -> str:
"""Map common backend exceptions to a short user-facing message."""
@@ -229,6 +302,7 @@ def create_connectors_router():
return "Connection timed out."
return raw
+ @_serialized_sync
def _start_sync(connector_id: str, instance: Any) -> str:
"""Spawn a background sync; returns ``"started"`` or ``"already_syncing"``.
@@ -238,11 +312,15 @@ def create_connectors_router():
inline. Both POST /connect (auto-ingest) and POST /sync (manual)
route through here so they share guard, state, and error handling.
"""
- import threading
-
- existing = _sync_threads.get(connector_id)
- if existing and existing.is_alive():
- return "already_syncing"
+ with _sync_lock:
+ existing = _sync_threads.get(connector_id)
+ if existing and existing.is_alive():
+ return "already_syncing"
+ # Detect a disconnect that completes while the checkpoint baseline
+ # below is being read, before this start has published its thread.
+ # Without a generation token that race can launch a writer after
+ # disconnect has already purged the connector's indexed content.
+ start_generation = _sync_generations.get(connector_id, 0)
# Snapshot the prior checkpoint count so the GET handler can
# report "X new this run" without each client tracking it.
@@ -252,19 +330,17 @@ def create_connectors_router():
from openjarvis.connectors.store import KnowledgeStore
from openjarvis.connectors.sync_engine import SyncEngine
- cp = SyncEngine(
- pipeline=IngestionPipeline(store=KnowledgeStore()),
- ).get_checkpoint(connector_id)
+ with KnowledgeStore() as store:
+ with SyncEngine(
+ pipeline=IngestionPipeline(store=store),
+ ) as engine:
+ cp = engine.get_checkpoint(connector_id)
if cp and cp.get("items_synced") is not None:
baseline_items = int(cp["items_synced"])
except Exception: # noqa: BLE001
baseline_items = 0
- _sync_state[connector_id] = {
- "state": "syncing",
- "error": None,
- "baseline_items": baseline_items,
- }
+ cancel_event = threading.Event()
def _run_sync() -> None:
try:
@@ -272,28 +348,61 @@ def create_connectors_router():
from openjarvis.connectors.store import KnowledgeStore
from openjarvis.connectors.sync_engine import SyncEngine
- store = KnowledgeStore()
- pipeline = IngestionPipeline(store=store)
- engine = SyncEngine(pipeline=pipeline)
- engine.sync(instance)
- logger.info("Sync completed for %s", connector_id)
- _sync_state[connector_id] = {
- "state": "complete",
- "error": None,
- "baseline_items": baseline_items,
- }
+ with KnowledgeStore() as store:
+ with SyncEngine(
+ pipeline=IngestionPipeline(store=store),
+ ) as engine:
+ engine.sync(instance, cancel_event=cancel_event)
+ final_state = "cancelled" if cancel_event.is_set() else "complete"
+ logger.info("Sync %s for %s", final_state, connector_id)
+ _publish_sync_state(
+ connector_id,
+ start_generation,
+ {
+ "state": final_state,
+ "error": None,
+ "baseline_items": baseline_items,
+ },
+ )
except Exception as exc:
+ if cancel_event.is_set():
+ _publish_sync_state(
+ connector_id,
+ start_generation,
+ {
+ "state": "cancelled",
+ "error": None,
+ "baseline_items": baseline_items,
+ },
+ )
+ return
error_msg = _translate_sync_error(str(exc))
logger.error("Sync failed for %s: %s", connector_id, error_msg)
- _sync_state[connector_id] = {
- "state": "error",
- "error": error_msg,
- "baseline_items": baseline_items,
- }
+ _publish_sync_state(
+ connector_id,
+ start_generation,
+ {
+ "state": "error",
+ "error": error_msg,
+ "baseline_items": baseline_items,
+ },
+ )
t = threading.Thread(target=_run_sync, daemon=True)
- t.start()
- _sync_threads[connector_id] = t
+ with _sync_lock:
+ if _sync_generations.get(connector_id, 0) != start_generation:
+ return "cancelled"
+ existing = _sync_threads.get(connector_id)
+ if existing and existing.is_alive():
+ return "already_syncing"
+ _sync_state[connector_id] = {
+ "state": "syncing",
+ "error": None,
+ "baseline_items": baseline_items,
+ }
+ _sync_cancel_events[connector_id] = cancel_event
+ _sync_threads[connector_id] = t
+ t.start()
return "started"
# ------------------------------------------------------------------
@@ -377,6 +486,7 @@ def create_connectors_router():
}
@router.post("/{connector_id}/connect")
+ @_serialized_async
async def connect_connector(connector_id: str, req: ConnectRequest):
"""Connect a connector using the supplied credentials."""
_ensure_connectors_registered()
@@ -385,6 +495,7 @@ def create_connectors_router():
status_code=404,
detail=f"Connector '{connector_id}' not found",
)
+ _reject_pending_disconnect(connector_id)
instance = _get_or_create(connector_id)
try:
@@ -484,6 +595,7 @@ def create_connectors_router():
}
@router.post("/{connector_id}/disconnect")
+ @_serialized_async
async def disconnect_connector(connector_id: str):
"""Disconnect a connector and clear its credentials."""
_ensure_connectors_registered()
@@ -493,10 +605,87 @@ def create_connectors_router():
detail=f"Connector '{connector_id}' not found",
)
instance = _get_or_create(connector_id)
+ with _sync_lock:
+ _sync_generations[connector_id] = _sync_generations.get(connector_id, 0) + 1
+ cancel_event = _sync_cancel_events.get(connector_id)
+ sync_thread = _sync_threads.get(connector_id)
+ if cancel_event is not None:
+ cancel_event.set()
+ if sync_thread is not None and sync_thread.is_alive():
+ previous_state = _sync_state.get(connector_id, {})
+ _sync_state[connector_id] = {
+ **previous_state,
+ "state": "stopping",
+ "error": None,
+ }
+
+ if sync_thread is not None and sync_thread.is_alive():
+ sync_thread.join(timeout=_SYNC_STOP_TIMEOUT_SECONDS)
+ if sync_thread is not None and sync_thread.is_alive():
+ raise HTTPException(
+ status_code=409,
+ detail=(
+ f"Sync for '{connector_id}' is still stopping; "
+ "indexed content was not purged"
+ ),
+ )
+
+ # Only revoke/delete credentials after the worker has relinquished
+ # the old connection. A timeout must leave the connector usable and
+ # reject reconnect attempts, rather than silently switching source
+ # state underneath a still-running writer.
try:
instance.disconnect()
except Exception as exc:
raise HTTPException(status_code=500, detail=str(exc))
+
+ # A source can be shared by multiple connector implementations
+ # (Gmail OAuth and Gmail IMAP both write source='gmail'). Preserve it
+ # while another owner is connected; otherwise purge it only after the
+ # in-flight writer has stopped.
+ purge_sources = set(_knowledge_sources(connector_id, instance))
+ for other_id in ConnectorRegistry.keys():
+ if other_id == connector_id:
+ continue
+ other_cls = ConnectorRegistry.get(other_id)
+ shared = purge_sources.intersection(_knowledge_sources(other_id, other_cls))
+ if not shared:
+ continue
+ try:
+ if _get_or_create(other_id).is_connected():
+ purge_sources.difference_update(shared)
+ except Exception:
+ # If ownership cannot be established safely, retain data.
+ purge_sources.difference_update(shared)
+
+ try:
+ from openjarvis.connectors.pipeline import IngestionPipeline
+ from openjarvis.connectors.store import KnowledgeStore
+ from openjarvis.connectors.sync_engine import SyncEngine
+
+ with KnowledgeStore() as store:
+ with SyncEngine(
+ pipeline=IngestionPipeline(store=store),
+ ) as engine:
+ old_checkpoint = engine.get_checkpoint(connector_id)
+ engine.reset_checkpoint(connector_id)
+ try:
+ store.delete_by_sources(purge_sources)
+ except Exception:
+ engine.restore_checkpoint(connector_id, old_checkpoint)
+ raise
+ except Exception as exc: # noqa: BLE001
+ logger.exception("Disconnect cleanup failed for %s", connector_id)
+ raise HTTPException(
+ status_code=500,
+ detail=f"Disconnected, but indexed-data cleanup failed: {exc}",
+ )
+
+ with _sync_lock:
+ _sync_cancel_events.pop(connector_id, None)
+ _sync_threads.pop(connector_id, None)
+ _sync_state.pop(connector_id, None)
+
return {
"connector_id": connector_id,
"connected": False,
@@ -551,6 +740,7 @@ def create_connectors_router():
return RedirectResponse(url=auth_url)
@router.get("/{connector_id}/oauth/callback")
+ @_serialized_async
async def oauth_callback(
connector_id: str,
request: Request,
@@ -570,6 +760,8 @@ def create_connectors_router():
_ensure_connectors_registered()
+ _reject_pending_disconnect(connector_id)
+
if error:
_style = "font-family:system-ui;text-align:center;padding:60px"
return HTMLResponse(
@@ -641,6 +833,7 @@ def create_connectors_router():
)
@router.post("/{connector_id}/sync")
+ @_serialized_sync
def trigger_sync(connector_id: str) -> Dict[str, Any]:
"""Trigger a sync in the background and return immediately."""
_ensure_connectors_registered()
@@ -650,6 +843,7 @@ def create_connectors_router():
detail=f"Connector '{connector_id}' not found",
)
inst = _get_or_create(connector_id)
+ _reject_pending_disconnect(connector_id)
if not inst.is_connected():
raise HTTPException(
status_code=400,
@@ -691,29 +885,22 @@ def create_connectors_router():
from openjarvis.connectors.store import KnowledgeStore
from openjarvis.connectors.sync_engine import SyncEngine
- store = KnowledgeStore()
- checkpoint = SyncEngine(
- pipeline=IngestionPipeline(store=store),
- ).get_checkpoint(connector_id)
-
- # Map connector_id → source field as written by the connector.
- # Most match 1:1, but the IMAP/OAuth Gmail connectors both write
- # source='gmail' so the unified card pulls a single timeline.
- _STORE_SOURCE = {"gmail_imap": "gmail"}
- store_source = _STORE_SOURCE.get(connector_id, connector_id)
-
- # Oldest indexed item for this connector so the UI can show how
- # far back the corpus reaches ("synced back to 2024-03-12").
- # ISO 8601 strings sort correctly under MIN(); skip rows with no
- # timestamp and any tombstoned chunks so the display reflects
- # what's actually retrievable.
- row = store._conn.execute(
- "SELECT MIN(timestamp) FROM knowledge_chunks "
- "WHERE source = ? AND timestamp != '' AND deleted_at IS NULL",
- (store_source,),
- ).fetchone()
- if row is not None and row[0]:
- oldest_item_date = str(row[0])
+ with KnowledgeStore() as store:
+ with SyncEngine(
+ pipeline=IngestionPipeline(store=store),
+ ) as engine:
+ checkpoint = engine.get_checkpoint(connector_id)
+
+ sources = _knowledge_sources(connector_id, instance)
+ placeholders = ", ".join("?" for _ in sources)
+ row = store._conn.execute(
+ "SELECT MIN(timestamp) FROM knowledge_chunks "
+ f"WHERE source IN ({placeholders}) AND timestamp != '' "
+ "AND deleted_at IS NULL",
+ sources,
+ ).fetchone()
+ if row is not None and row[0]:
+ oldest_item_date = str(row[0])
except Exception: # noqa: BLE001
checkpoint = None
@@ -741,7 +928,9 @@ def create_connectors_router():
last_sync_str = None
# Determine effective state
- if is_bg_running:
+ if _disconnect_pending(connector_id):
+ effective_state = "stopping"
+ elif is_bg_running:
effective_state = "syncing"
elif bg.get("state") == "error":
effective_state = "error"
diff --git a/tests/connectors/test_store.py b/tests/connectors/test_store.py
index 5ecc06aa..eee32275 100644
--- a/tests/connectors/test_store.py
+++ b/tests/connectors/test_store.py
@@ -202,6 +202,58 @@ def test_delete_by_doc_id(ks: KnowledgeStore) -> None:
assert ks.delete("nonexistent:doc") is False
+def test_delete_by_source(ks: KnowledgeStore) -> None:
+ """delete_by_source() removes only chunks from the given source.
+
+ Regression: reconnecting a filesystem connector (e.g. Obsidian) to a
+ different underlying vault left the OLD vault's chunks in the store
+ forever. If the new vault happened to contain a file at the same
+ relative path (e.g. both vaults have a "Welcome.md"), the resulting
+ doc_id collided with the old, orphaned chunk -- and the ingestion
+ pipeline's duplicate-doc_id dedup silently discarded the new content,
+ leaving permanently stale data indexed with no error surfaced.
+ Disconnecting a connector must purge its previously-ingested content
+ so a later reconnect (to the same or a different source) starts clean.
+ """
+ _store(
+ ks,
+ content="Old vault welcome note content",
+ source="obsidian",
+ doc_id="obsidian:Welcome.md",
+ )
+ _store(
+ ks,
+ content="Unrelated note from another connector",
+ source="notion",
+ doc_id="notion:Some Page",
+ )
+
+ deleted = ks.delete_by_source("obsidian")
+ assert deleted == 1
+
+ after = ks.retrieve("welcome", top_k=10)
+ for r in after:
+ assert r.metadata.get("source") != "obsidian"
+
+ # The other source's content must be untouched.
+ remaining = ks.retrieve("unrelated note connector", top_k=10)
+ assert any(r.metadata.get("source") == "notion" for r in remaining)
+
+ # No matching source is a no-op, not an error.
+ assert ks.delete_by_source("nonexistent_source") == 0
+
+
+def test_delete_by_sources_removes_all_owned_sources_atomically(
+ ks: KnowledgeStore,
+) -> None:
+ _store(ks, content="first owned source", source="owned_a")
+ _store(ks, content="second owned source", source="owned_b")
+ _store(ks, content="unrelated source", source="other")
+
+ assert ks.delete_by_sources(("owned_a", "owned_b")) == 2
+ assert ks.distinct_sources() == ["other"]
+
+
def test_clear(ks: KnowledgeStore) -> None:
"""clear() removes all stored documents."""
_store(ks, content="Document one about research")
diff --git a/tests/connectors/test_sync_engine.py b/tests/connectors/test_sync_engine.py
index 8d7d1e9d..eb1f7b54 100644
--- a/tests/connectors/test_sync_engine.py
+++ b/tests/connectors/test_sync_engine.py
@@ -2,6 +2,8 @@
from __future__ import annotations
+import sqlite3
+import threading
from datetime import datetime, timezone
from pathlib import Path
from typing import Iterator, List, Optional
@@ -225,7 +227,94 @@ def test_sync_multiple_connectors(
# ---------------------------------------------------------------------------
-# Test 5: a failed sync must not advance last_sync (#782)
+# reset_checkpoint clears stale state after a data-source swap
+# ---------------------------------------------------------------------------
+
+
+def test_reset_checkpoint_clears_state(engine: SyncEngine) -> None:
+ """A connector reconfigured to point at an entirely different data
+ source (e.g. a user swaps which Obsidian vault is connected) must not
+ resume from the old source's checkpoint. reset_checkpoint() clears the
+ saved cursor/items_synced/last_sync for a connector_id so the next
+ sync starts fresh."""
+ docs = [_make_doc(f"reset:doc:{i}") for i in range(5)]
+ connector = StubConnector(docs)
+
+ engine.sync(connector)
+ assert engine.get_checkpoint("stub") is not None
+
+ engine.reset_checkpoint("stub")
+
+ assert engine.get_checkpoint("stub") is None
+
+
+def test_sync_after_reset_does_not_inherit_prior_items_or_since(
+ engine: SyncEngine,
+) -> None:
+ """After reset_checkpoint, a new sync must start item counting from
+ zero and pass since=None -- not the old source's watermark, which
+ could cause new items from a freshly-connected source to be silently
+ skipped if their timestamps predate it."""
+
+ class TrackingConnector(StubConnector):
+ connector_id = "swap"
+
+ def __init__(self, docs: List[Document]) -> None:
+ super().__init__(docs)
+ self.received_since: List[Optional[datetime]] = []
+
+ def sync(
+ self,
+ *,
+ since: Optional[datetime] = None,
+ cursor: Optional[str] = None,
+ ) -> Iterator[Document]:
+ self.received_since.append(since)
+ yield from self._docs
+
+ old_docs = [_make_doc(f"old:doc:{i}") for i in range(9)]
+ engine.sync(TrackingConnector(old_docs))
+
+ engine.reset_checkpoint("swap")
+
+ new_docs = [_make_doc(f"new:doc:{i}") for i in range(1)]
+ fresh = TrackingConnector(new_docs)
+ items = engine.sync(fresh)
+
+ assert fresh.received_since == [None]
+ assert items == 1
+
+ cp = engine.get_checkpoint("swap")
+ assert cp is not None
+ assert cp["items_synced"] == 1
+
+
+def test_sync_honors_cancellation_before_ingestion(
+ engine: SyncEngine,
+ store: KnowledgeStore,
+) -> None:
+ cancel_event = threading.Event()
+ cancel_event.set()
+
+ items = engine.sync(
+ StubConnector([_make_doc("cancelled:doc")]),
+ cancel_event=cancel_event,
+ )
+
+ assert items == 0
+ assert store.count() == 0
+
+
+def test_sync_engine_context_manager_closes_sqlite(pipeline: IngestionPipeline) -> None:
+ with SyncEngine(pipeline, state_db=":memory:") as scoped_engine:
+ scoped_engine.get_checkpoint("stub")
+
+ with pytest.raises(sqlite3.ProgrammingError):
+ scoped_engine.get_checkpoint("stub")
+
+
+# ---------------------------------------------------------------------------
+# A failed or cancelled sync must not advance last_sync (#782)
# ---------------------------------------------------------------------------
@@ -271,3 +360,30 @@ def test_retry_after_failure_does_not_skip_the_failed_window(
assert cp is not None
assert cp["last_sync"] is not None
assert cp["error"] is None
+
+
+def test_cancelled_sync_does_not_advance_last_sync(
+ engine: SyncEngine,
+ store: KnowledgeStore,
+) -> None:
+ """Disconnect cancellation is incomplete work, not a successful sync."""
+ cancel_event = threading.Event()
+
+ class CancellingConnector(StubConnector):
+ connector_id = "cancel-watermark"
+
+ def sync(self, **kwargs):
+ yield _make_doc("kept")
+ cancel_event.set()
+ yield _make_doc("not-ingested")
+
+ items = engine.sync(
+ CancellingConnector([]),
+ cancel_event=cancel_event,
+ )
+
+ assert items == 0
+ assert store.count() == 0
+ cp = engine.get_checkpoint("cancel-watermark")
+ assert cp is not None
+ assert cp["last_sync"] is None
diff --git a/tests/server/test_connectors_router.py b/tests/server/test_connectors_router.py
index a736fa06..b5526086 100644
--- a/tests/server/test_connectors_router.py
+++ b/tests/server/test_connectors_router.py
@@ -2,6 +2,8 @@
from __future__ import annotations
+import threading
+import time
from pathlib import Path
import pytest
@@ -77,6 +79,401 @@ def test_disconnect(app):
assert data["connected"] is False
+def test_disconnect_clears_stale_sync_checkpoint(app) -> None:
+ """Disconnecting a connector must reset its sync checkpoint.
+
+ Regression: a user disconnects Obsidian from one vault and reconnects
+ it to a different vault. Before this fix, the SyncEngine checkpoint
+ (items_synced, cursor, last_sync) was keyed only by connector_id and
+ survived disconnect, so the next sync resumed from the OLD vault's
+ watermark -- inflating the reported item count with stale data and
+ risking skipped items in the new vault whose timestamps predate the
+ old watermark.
+ """
+ from openjarvis.connectors.pipeline import IngestionPipeline
+ from openjarvis.connectors.store import KnowledgeStore
+ from openjarvis.connectors.sync_engine import SyncEngine
+
+ with KnowledgeStore() as store:
+ with SyncEngine(pipeline=IngestionPipeline(store=store)) as engine:
+ engine._save_checkpoint("obsidian", 9, cursor="stale-cursor")
+ assert engine.get_checkpoint("obsidian") is not None
+
+ resp = app.post("/v1/connectors/obsidian/disconnect")
+ assert resp.status_code == 200
+
+ # A fresh SyncEngine (same default state DB) must see no checkpoint.
+ with KnowledgeStore() as store:
+ with SyncEngine(pipeline=IngestionPipeline(store=store)) as fresh_engine:
+ assert fresh_engine.get_checkpoint("obsidian") is None
+
+
+def test_disconnect_purges_previously_ingested_content(app) -> None:
+ """Disconnecting a connector must purge its previously-ingested chunks.
+
+ Regression: reconnecting Obsidian to a different vault left the OLD
+ vault's chunks in the KnowledgeStore forever (disconnect only cleared
+ credentials, never indexed content). If the new vault contains a file
+ at the same relative path as one in the old vault (e.g. both have a
+ "Welcome.md"), the resulting doc_id collides with the orphaned old
+ chunk, and the ingestion pipeline's duplicate-doc_id dedup silently
+ discards the new content -- the user's real notes never get indexed,
+ with no error surfaced anywhere.
+ """
+ from openjarvis.connectors.store import KnowledgeStore
+
+ with KnowledgeStore() as store:
+ store.store(
+ content="Stale content from a previously-connected vault",
+ source="obsidian",
+ doc_type="note",
+ doc_id="obsidian:Welcome.md",
+ title="Welcome",
+ author="",
+ )
+ assert any(
+ r.metadata.get("source") == "obsidian"
+ for r in store.retrieve("stale content vault", top_k=10)
+ )
+
+ resp = app.post("/v1/connectors/obsidian/disconnect")
+ assert resp.status_code == 200
+
+ with KnowledgeStore() as fresh_store:
+ assert not any(
+ r.metadata.get("source") == "obsidian"
+ for r in fresh_store.retrieve("stale content vault", top_k=10)
+ )
+
+
+def test_disconnect_cancels_inflight_sync_before_purge(app) -> None:
+ from openjarvis.connectors._stubs import Document, SyncStatus
+ from openjarvis.connectors.store import KnowledgeStore
+ from openjarvis.server.connectors_router import _instances
+
+ started = threading.Event()
+ released = threading.Event()
+ disconnect_called = threading.Event()
+
+ class BlockingConnector:
+ connector_id = "obsidian"
+ indexed_sources = ("obsidian",)
+
+ def is_connected(self):
+ return True
+
+ def sync(self, **kwargs):
+ started.set()
+ released.wait(timeout=2)
+ yield Document(
+ doc_id="obsidian:late-write",
+ source="obsidian",
+ doc_type="note",
+ content="must not survive disconnect",
+ )
+
+ def disconnect(self):
+ disconnect_called.set()
+
+ def sync_status(self):
+ return SyncStatus()
+
+ _instances["obsidian"] = BlockingConnector()
+ responses = []
+
+ def disconnect_request():
+ responses.append(app.post("/v1/connectors/obsidian/disconnect"))
+
+ disconnect_thread = threading.Thread(target=disconnect_request)
+ try:
+ assert app.post("/v1/connectors/obsidian/sync").status_code == 200
+ assert started.wait(timeout=2)
+ disconnect_thread.start()
+
+ deadline = time.monotonic() + 2
+ while time.monotonic() < deadline:
+ if app.get("/v1/connectors/obsidian/sync").json()["state"] == "stopping":
+ break
+ time.sleep(0.01)
+ else:
+ pytest.fail("disconnect never entered the stopping state")
+
+ # Credentials remain intact until the old writer has stopped.
+ assert not disconnect_called.is_set()
+ released.set()
+ disconnect_thread.join(timeout=3)
+ assert not disconnect_thread.is_alive()
+ assert responses[0].status_code == 200
+ assert disconnect_called.is_set()
+ with KnowledgeStore() as store:
+ assert not any(
+ result.metadata.get("doc_id") == "obsidian:late-write"
+ for result in store.retrieve("must survive disconnect", top_k=10)
+ )
+ finally:
+ released.set()
+ disconnect_thread.join(timeout=3)
+ _instances.pop("obsidian", None)
+
+
+def test_disconnect_timeout_preserves_source_and_guards_reconnect(
+ app,
+ monkeypatch,
+ tmp_path,
+) -> None:
+ from openjarvis.connectors._stubs import SyncStatus
+ from openjarvis.server import connectors_router
+ from openjarvis.server.connectors_router import _instances
+
+ started = threading.Event()
+ released = threading.Event()
+ finished = threading.Event()
+ disconnect_called = threading.Event()
+
+ class StuckConnector:
+ connector_id = "obsidian"
+ indexed_sources = ("obsidian",)
+ auth_type = "filesystem"
+
+ def __init__(self):
+ self._connected = True
+ self._vault_path = "old-vault"
+
+ def is_connected(self):
+ return self._connected
+
+ def sync(self, **kwargs):
+ started.set()
+ released.wait(timeout=3)
+ finished.set()
+ if False:
+ yield
+
+ def disconnect(self):
+ disconnect_called.set()
+ self._connected = False
+
+ def sync_status(self):
+ return SyncStatus()
+
+ connector = StuckConnector()
+ _instances["obsidian"] = connector
+ monkeypatch.setattr(connectors_router, "_SYNC_STOP_TIMEOUT_SECONDS", 0.05)
+ new_vault = tmp_path / "new-vault"
+ new_vault.mkdir()
+
+ try:
+ assert app.post("/v1/connectors/obsidian/sync").status_code == 200
+ assert started.wait(timeout=2)
+
+ response = app.post("/v1/connectors/obsidian/disconnect")
+ assert response.status_code == 409
+ assert connector.is_connected()
+ assert connector._vault_path == "old-vault"
+ assert not disconnect_called.is_set()
+
+ reconnect = app.post(
+ "/v1/connectors/obsidian/connect",
+ json={"path": str(new_vault)},
+ )
+ assert reconnect.status_code == 409
+ assert connector._vault_path == "old-vault"
+ assert app.post("/v1/connectors/obsidian/sync").status_code == 409
+
+ released.set()
+ assert finished.wait(timeout=2)
+ assert app.get("/v1/connectors/obsidian/sync").json()["state"] == "stopping"
+ response = app.post("/v1/connectors/obsidian/disconnect")
+ assert response.status_code == 200
+ assert disconnect_called.is_set()
+ assert not connector.is_connected()
+ finally:
+ released.set()
+ _instances.pop("obsidian", None)
+
+
+def test_disconnect_prevents_sync_start_racing_checkpoint_read(
+ app,
+ monkeypatch,
+) -> None:
+ """Disconnect waits for an in-progress start decision, then purges it."""
+ from openjarvis.connectors._stubs import Document, SyncStatus
+ from openjarvis.connectors.sync_engine import SyncEngine
+ from openjarvis.server.connectors_router import _instances
+
+ baseline_started = threading.Event()
+ release_baseline = threading.Event()
+ sync_called = threading.Event()
+ gate_lock = threading.Lock()
+ gate_first_call = True
+ original_get_checkpoint = SyncEngine.get_checkpoint
+
+ def gated_get_checkpoint(self, connector_id):
+ nonlocal gate_first_call
+ with gate_lock:
+ should_gate = gate_first_call
+ gate_first_call = False
+ if should_gate:
+ baseline_started.set()
+ assert release_baseline.wait(timeout=3)
+ return original_get_checkpoint(self, connector_id)
+
+ monkeypatch.setattr(SyncEngine, "get_checkpoint", gated_get_checkpoint)
+
+ class RacingConnector:
+ connector_id = "obsidian"
+ indexed_sources = ("obsidian",)
+
+ def __init__(self):
+ self.connected = True
+
+ def is_connected(self):
+ return self.connected
+
+ def sync(self, **kwargs):
+ sync_called.set()
+ yield Document(
+ doc_id="obsidian:post-disconnect",
+ source="obsidian",
+ doc_type="note",
+ content="must never be written",
+ )
+
+ def disconnect(self):
+ self.connected = False
+
+ def sync_status(self):
+ return SyncStatus()
+
+ _instances["obsidian"] = RacingConnector()
+ sync_response = []
+ disconnect_response = []
+ disconnect_done = threading.Event()
+
+ def trigger_sync():
+ sync_response.append(app.post("/v1/connectors/obsidian/sync"))
+
+ def disconnect():
+ disconnect_response.append(app.post("/v1/connectors/obsidian/disconnect"))
+ disconnect_done.set()
+
+ request_thread = threading.Thread(target=trigger_sync)
+ disconnect_thread = threading.Thread(target=disconnect)
+ try:
+ request_thread.start()
+ assert baseline_started.wait(timeout=3)
+
+ disconnect_thread.start()
+ # The checkpoint snapshot and thread publication are one lifecycle
+ # operation. Disconnect must not overtake it and purge too early.
+ assert not disconnect_done.wait(timeout=0.1)
+
+ release_baseline.set()
+ request_thread.join(timeout=3)
+ disconnect_thread.join(timeout=3)
+ assert not request_thread.is_alive()
+ assert not disconnect_thread.is_alive()
+ assert sync_response[0].status_code == 200
+ assert sync_response[0].json()["status"] == "started"
+ assert disconnect_response[0].status_code == 200
+
+ from openjarvis.connectors.store import KnowledgeStore
+
+ with KnowledgeStore() as store:
+ assert not any(
+ result.metadata.get("doc_id") == "obsidian:post-disconnect"
+ for result in store.retrieve("must never be written", top_k=10)
+ )
+ finally:
+ release_baseline.set()
+ request_thread.join(timeout=3)
+ disconnect_thread.join(timeout=3)
+ _instances.pop("obsidian", None)
+
+
+def test_disconnect_preserves_source_owned_by_connected_peer(app) -> None:
+ from openjarvis.connectors._stubs import SyncStatus
+ from openjarvis.connectors.store import KnowledgeStore
+ from openjarvis.server.connectors_router import _instances
+
+ class FakeConnector:
+ def __init__(self, connector_id, indexed_sources=()):
+ self.connector_id = connector_id
+ self.indexed_sources = indexed_sources
+ self.connected = True
+
+ def is_connected(self):
+ return self.connected
+
+ def disconnect(self):
+ self.connected = False
+
+ def sync_status(self):
+ return SyncStatus()
+
+ _instances["gmail"] = FakeConnector("gmail")
+ _instances["gmail_imap"] = FakeConnector("gmail_imap", ("gmail",))
+ try:
+ with KnowledgeStore() as store:
+ store.store(
+ content="shared gmail ownership sentinel",
+ source="gmail",
+ doc_type="email",
+ doc_id="gmail:shared-owner",
+ )
+
+ response = app.post("/v1/connectors/gmail_imap/disconnect")
+ assert response.status_code == 200
+
+ with KnowledgeStore() as store:
+ assert any(
+ result.metadata.get("doc_id") == "gmail:shared-owner"
+ for result in store.retrieve("ownership sentinel", top_k=10)
+ )
+ store.delete_by_source("gmail")
+ finally:
+ _instances.pop("gmail", None)
+ _instances.pop("gmail_imap", None)
+
+
+def test_disconnect_restores_checkpoint_when_purge_fails(app, monkeypatch) -> None:
+ from openjarvis.connectors.pipeline import IngestionPipeline
+ from openjarvis.connectors.store import KnowledgeStore
+ from openjarvis.connectors.sync_engine import SyncEngine
+ from openjarvis.server.connectors_router import _instances
+
+ class FakeObsidian:
+ indexed_sources = ("obsidian",)
+
+ def is_connected(self):
+ return True
+
+ def disconnect(self):
+ pass
+
+ with KnowledgeStore() as store:
+ with SyncEngine(pipeline=IngestionPipeline(store=store)) as engine:
+ engine._save_checkpoint("obsidian", 17, cursor="keep-me")
+
+ def fail_purge(self, sources):
+ raise RuntimeError("simulated purge failure")
+
+ monkeypatch.setattr(KnowledgeStore, "delete_by_sources", fail_purge)
+ _instances["obsidian"] = FakeObsidian()
+ try:
+ response = app.post("/v1/connectors/obsidian/disconnect")
+ assert response.status_code == 500
+ with KnowledgeStore() as store:
+ with SyncEngine(pipeline=IngestionPipeline(store=store)) as engine:
+ checkpoint = engine.get_checkpoint("obsidian")
+ assert checkpoint is not None
+ assert checkpoint["items_synced"] == 17
+ assert checkpoint["cursor"] == "keep-me"
+ engine.reset_checkpoint("obsidian")
+ finally:
+ _instances.pop("obsidian", None)
+
+
def test_sync_status(app):
"""GET /v1/connectors/obsidian/sync returns a response with a state field."""
resp = app.get("/v1/connectors/obsidian/sync")