Skip to content

Personal AI, On Personal Devices

Python
10.2 KB
Raw Blame
"""SyncEngine — checkpoint/resume orchestration for connector syncs.

Wraps ``IngestionPipeline`` with a lightweight SQLite state database so that
long-running syncs can be interrupted and resumed from the last saved cursor.

Typical usage::

    store = KnowledgeStore(db_path=":memory:")
    pipeline = IngestionPipeline(store)
    engine = SyncEngine(pipeline)

    items = engine.sync(connector)        # first run
    items = engine.sync(connector)        # resumes from saved cursor
    cp = engine.get_checkpoint(connector.connector_id)
"""

from __future__ import annotations

import sqlite3
import threading
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, Optional

from openjarvis.connectors._stubs import BaseConnector
from openjarvis.connectors.pipeline import IngestionPipeline
from openjarvis.core.config import DEFAULT_CONFIG_DIR

# ---------------------------------------------------------------------------
# DDL
# ---------------------------------------------------------------------------

_CREATE_STATE_TABLE = """
CREATE TABLE IF NOT EXISTS sync_state (
    connector_id  TEXT PRIMARY KEY,
    items_synced  INTEGER NOT NULL DEFAULT 0,
    cursor        TEXT,
    last_sync     TEXT,
    error         TEXT
);
"""

_BATCH_SIZE = 100


class SyncEngine:
    """Orchestrate connector syncs with checkpoint/resume tracking.

    Parameters
    ----------
    pipeline:
        The ``IngestionPipeline`` that documents are fed into.
    state_db:
        Path to the SQLite database used for checkpoint state.  If empty,
        defaults to ``DEFAULT_CONFIG_DIR / "sync_state.db"``.
    """

    def __init__(self, pipeline: IngestionPipeline, *, state_db: str = "") -> None:
        self._pipeline = pipeline

        if not state_db:
            db_path = DEFAULT_CONFIG_DIR / "sync_state.db"
        else:
            db_path = Path(state_db)

        # Ensure parent directory exists (skip for :memory:)
        if str(db_path) != ":memory:":
            db_path.parent.mkdir(parents=True, exist_ok=True)

        self._conn = sqlite3.connect(str(db_path), check_same_thread=False)
        self._conn.row_factory = sqlite3.Row
        self._conn.execute("PRAGMA journal_mode=WAL;")
        self._conn.execute(_CREATE_STATE_TABLE)
        self._conn.commit()

    def __enter__(self) -> "SyncEngine":
        return self

    def __exit__(self, *exc_info: object) -> None:
        self.close()

    # ------------------------------------------------------------------
    # Public API
    # ------------------------------------------------------------------

    def sync(
        self,
        connector: BaseConnector,
        *,
        cancel_event: Optional[threading.Event] = None,
    ) -> int:
        """Run a full sync for *connector* and return the number of items ingested.

        Resumes from the last saved cursor if one exists.  Documents are
        batched in groups of 100 before being handed to the pipeline; a
        checkpoint is saved after every batch and once more at the end.

        On error the checkpoint is updated with the error message and the
        exception is re-raised so callers can handle it.
        """
        connector_id: str = connector.connector_id

        # Load any previous checkpoint so we can resume.
        checkpoint = self.get_checkpoint(connector_id)
        prior_cursor: Optional[str] = checkpoint["cursor"] if checkpoint else None
        prior_items: int = checkpoint["items_synced"] if checkpoint else 0

        since: Optional[datetime] = None
        if checkpoint and checkpoint.get("last_sync"):
            try:
                since = datetime.fromisoformat(checkpoint["last_sync"])
            except (ValueError, TypeError):
                pass

        # Captured before the first fetch, not at completion, so items
        # created/modified while this sync was running are still covered
        # by the *next* sync's `since` filter instead of being skipped.
        sync_started_at = datetime.now(tz=timezone.utc).isoformat()

        items_ingested = 0
        current_cursor: Optional[str] = prior_cursor

        try:
            doc_iter = connector.sync(since=since, cursor=prior_cursor)

            batch = []
            for doc in doc_iter:
                if cancel_event is not None and cancel_event.is_set():
                    break
                batch.append(doc)

                if len(batch) >= _BATCH_SIZE:
                    items_ingested += self._pipeline.ingest(batch)
                    batch = []
                    # Progress checkpoint: track cursor/items so a later
                    # retry can resume from here, but must NOT advance
                    # last_sync -- this sync hasn't completed yet, and
                    # advancing the watermark on a checkpoint that isn't a
                    # successful completion would permanently skip
                    # whatever wasn't reached if the sync then fails (#782).
                    self._save_checkpoint(
                        connector_id,
                        prior_items + items_ingested,
                        cursor=current_cursor,
                    )

            # Ingest any remaining documents.
            if batch and not (cancel_event is not None and cancel_event.is_set()):
                items_ingested += self._pipeline.ingest(batch)

        except Exception as exc:
            self._save_checkpoint(
                connector_id,
                prior_items + items_ingested,
                cursor=current_cursor,
                error=str(exc),
            )
            raise

        if cancel_event is not None and cancel_event.is_set():
            # Cancellation means the source was not exhausted. Preserve the
            # prior successful watermark exactly as failure checkpoints do;
            # advancing it here could make the next sync skip documents that
            # the cancelled run never reached (#782).
            self._save_checkpoint(
                connector_id,
                prior_items + items_ingested,
                cursor=current_cursor,
                error=None,
            )
            return items_ingested

        # Final checkpoint on successful completion — clear any previous
        # error and advance the watermark to when this sync started.
        self._save_checkpoint(
            connector_id,
            prior_items + items_ingested,
            cursor=current_cursor,
            error=None,
            last_sync=sync_started_at,
        )
        return items_ingested

    def close(self) -> None:
        """Close the checkpoint database connection."""
        self._conn.close()

    def get_checkpoint(self, connector_id: str) -> Optional[Dict[str, Any]]:
        """Return the last checkpoint, or ``None`` if never synced."""
        sql = (
            "SELECT items_synced, cursor, last_sync, error"
            " FROM sync_state WHERE connector_id = ?"
        )
        row = self._conn.execute(sql, (connector_id,)).fetchone()

        if row is None:
            return None

        return {
            "items_synced": row["items_synced"],
            "cursor": row["cursor"],
            "last_sync": row["last_sync"],
            "error": row["error"],
        }

    def reset_checkpoint(self, connector_id: str) -> None:
        """Clear the saved checkpoint for *connector_id*.

        Must be called whenever a connector is reconfigured to point at a
        different underlying data source (e.g. disconnect/reconnect with a
        new Obsidian vault path). Without this, the next sync would resume
        from the old source's cursor/``since`` watermark, inflating
        ``items_synced`` with the old source's count and potentially
        skipping new items whose timestamps predate that watermark.
        """
        self._conn.execute(
            "DELETE FROM sync_state WHERE connector_id = ?", (connector_id,)
        )
        self._conn.commit()

    def restore_checkpoint(
        self,
        connector_id: str,
        checkpoint: Optional[Dict[str, Any]],
    ) -> None:
        """Restore an exact checkpoint snapshot after a failed cleanup."""
        if checkpoint is None:
            self.reset_checkpoint(connector_id)
            return
        self._conn.execute(
            """
            INSERT INTO sync_state
                (connector_id, items_synced, cursor, last_sync, error)
            VALUES (?, ?, ?, ?, ?)
            ON CONFLICT(connector_id) DO UPDATE SET
                items_synced = excluded.items_synced,
                cursor       = excluded.cursor,
                last_sync    = excluded.last_sync,
                error        = excluded.error
            """,
            (
                connector_id,
                checkpoint["items_synced"],
                checkpoint.get("cursor"),
                checkpoint.get("last_sync"),
                checkpoint.get("error"),
            ),
        )
        self._conn.commit()

    # ------------------------------------------------------------------
    # Internal helpers
    # ------------------------------------------------------------------

    def _save_checkpoint(
        self,
        connector_id: str,
        items_synced: int,
        *,
        cursor: Optional[str] = None,
        error: Optional[str] = None,
        last_sync: Optional[str] = None,
    ) -> None:
        """UPSERT a checkpoint row into the state database.

        ``last_sync`` only advances when the caller explicitly passes a
        value (a successful sync completion). ``COALESCE`` preserves the
        existing watermark on progress/error checkpoints, so a failed or
        still-in-progress sync can never permanently skip unprocessed
        items (#782).
        """
        self._conn.execute(
            """
            INSERT INTO sync_state
                (connector_id, items_synced, cursor, last_sync, error)
            VALUES (?, ?, ?, ?, ?)
            ON CONFLICT(connector_id) DO UPDATE SET
                items_synced = excluded.items_synced,
                cursor       = excluded.cursor,
                last_sync    = COALESCE(excluded.last_sync, sync_state.last_sync),
                error        = excluded.error
            """,
            (connector_id, items_synced, cursor, last_sync, error),
        )
        self._conn.commit()


__all__ = ["SyncEngine"]