Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

Preserve failed web source scans and renew fenced sync leases after #1385

Bingxi Zhao (Frank)46c9c885f7fc
diff --git a/deeptutor/services/web_source/repository.py b/deeptutor/services/web_source/repository.py
index c1e8be855..13828fc32 100644
--- a/deeptutor/services/web_source/repository.py
+++ b/deeptutor/services/web_source/repository.py
@@ -110,8 +110,13 @@ class SQLiteWebSourceSyncRepository:
def _now_ms(self) -> int:
return int(time.time() * 1000)
- def reconcile_sources(self, source_keys: set[tuple[str, str, str]]) -> None:
- """Create missing rows and remove jobs whose source no longer exists."""
+ def reconcile_sources(
+ self,
+ source_keys: set[tuple[str, str, str]],
+ *,
+ scanned_owner_ids: set[str] | None = None,
+ ) -> None:
+ """Reconcile only owners whose source inventory was read successfully."""
now = self._now_ms()
with self._connect() as connection:
connection.execute("BEGIN IMMEDIATE")
@@ -134,6 +139,8 @@ class SQLiteWebSourceSyncRepository:
(owner_id, kb_name, source_id, now, now),
)
for owner_id, kb_name, source_id in existing - source_keys:
+ if scanned_owner_ids is not None and owner_id not in scanned_owner_ids:
+ continue
connection.execute(
"""
DELETE FROM web_source_sync_jobs
@@ -260,27 +267,44 @@ class SQLiteWebSourceSyncRepository:
connection.execute("ROLLBACK")
raise
+ def renew_lease(self, job: WebSourceSyncJob, lease_until_ms: int) -> bool:
+ """Extend only the still-owned, unexpired claim."""
+ now = self._now_ms()
+ with self._connect() as connection:
+ cursor = connection.execute(
+ """
+ UPDATE web_source_sync_jobs
+ SET lease_until_ms=?, updated_at_ms=?
+ WHERE owner_id=? AND kb_name=? AND source_id=?
+ AND state='running' AND runner_id=? AND lease_until_ms>?
+ """,
+ (lease_until_ms, now, *self.key(job), job.runner_id, now),
+ )
+ return int(cursor.rowcount or 0) == 1
+
def _update(
self,
job: WebSourceSyncJob,
**fields: Any,
- ) -> None:
+ ) -> bool:
assignments = list(fields)
if not assignments:
- return
+ return False
fields["updated_at_ms"] = self._now_ms()
assignments.append("updated_at_ms")
values = list(fields.values())
set_sql = ", ".join(f"{name}=?" for name in assignments)
with self._connect() as connection:
- connection.execute(
+ cursor = connection.execute(
f"""
UPDATE web_source_sync_jobs
SET {set_sql}
WHERE owner_id=? AND kb_name=? AND source_id=?
+ AND state='running' AND runner_id=? AND lease_until_ms>?
""",
- (*values, *self.key(job)),
+ (*values, *self.key(job), job.runner_id, self._now_ms()),
)
+ return int(cursor.rowcount or 0) == 1
def mark_success(self, job: WebSourceSyncJob, next_run_at_ms: int) -> None:
self._update(
@@ -314,6 +338,17 @@ class SQLiteWebSourceSyncRepository:
lease_until_ms=None,
)
+ def mark_interrupted(self, job: WebSourceSyncJob) -> None:
+ self._update(
+ job,
+ state="interrupted",
+ next_run_at_ms=self._now_ms(),
+ error="Synchronization was interrupted",
+ cancel_requested=False,
+ runner_id="",
+ lease_until_ms=None,
+ )
+
def mark_cancelled(self, job: WebSourceSyncJob, next_run_at_ms: int | None = None) -> None:
now = self._now_ms()
with self._connect() as connection:
@@ -324,8 +359,9 @@ class SQLiteWebSourceSyncRepository:
lease_until_ms=NULL, last_run_at_ms=?, updated_at_ms=?,
next_run_at_ms=COALESCE(?, next_run_at_ms)
WHERE owner_id=? AND kb_name=? AND source_id=?
+ AND state='running' AND runner_id=? AND lease_until_ms>?
""",
- (now, now, next_run_at_ms, *self.key(job)),
+ (now, now, next_run_at_ms, *self.key(job), job.runner_id, now),
)
def request_cancel(self, job_key: tuple[str, str, str]) -> bool:
@@ -395,6 +431,7 @@ class SQLiteWebSourceSyncRepository:
cancel_requested=0, runner_id='', lease_until_ms=NULL,
updated_at_ms=?
WHERE owner_id=? AND kb_name=? AND source_id=?
+ AND state IN ('error', 'interrupted', 'cancelled')
""",
(now, now, *job_key),
)
diff --git a/deeptutor/services/web_source/scheduler.py b/deeptutor/services/web_source/scheduler.py
index 84172c585..fdcac9500 100644
--- a/deeptutor/services/web_source/scheduler.py
+++ b/deeptutor/services/web_source/scheduler.py
@@ -20,7 +20,8 @@ WEB_SYNC_MIN_INTERVAL_HOURS = 1
WEB_SYNC_MAX_INTERVAL_HOURS = 168
WEB_SYNC_INTERVAL_HOURS = 24
WEB_SYNC_CHECK_SECONDS = 15
-WEB_SYNC_LEASE_HOURS = 6
+WEB_SYNC_LEASE_SECONDS = 60
+WEB_SYNC_LEASE_RENEW_SECONDS = 20
WEB_SYNC_MAX_CONCURRENCY = 2
@@ -108,6 +109,7 @@ class WebSourceSyncScheduler:
while self._running:
try:
await self._synchronize_sources()
+ await asyncio.to_thread(self.repo.recover_interrupted, self._runner_id)
self._start_due_jobs()
except Exception:
logger.exception("Web source scheduler cycle failed")
@@ -150,30 +152,33 @@ class WebSourceSyncScheduler:
del manager
source_keys: set[tuple[str, str, str]] = set()
sources: dict[tuple[str, str, str], dict[str, Any]] = {}
+ scanned_owner_ids: set[str] = set()
for owner_id in owner_ids:
+ owner_sources: dict[tuple[str, str, str], dict[str, Any]] = {}
try:
owner_manager = await asyncio.to_thread(self._manager_for_owner, owner_id)
for kb_name, source in owner_manager.get_all_web_sources():
- if not source.get("enabled", True):
- continue
- if not source.get("auto_sync_enabled", True):
+ if not source.get("enabled", True) or not source.get("auto_sync_enabled", True):
continue
source_id = str(source.get("id") or "")
kb_key = str(kb_name)
- if not source_id or not kb_key:
- continue
- key = owner_id, kb_key, source_id
- source_keys.add(key)
- sources[key] = dict(source)
+ if source_id and kb_key:
+ owner_sources[owner_id, kb_key, source_id] = dict(source)
except Exception:
logger.exception("Failed to enumerate web sources for owner %s", owner_id)
+ continue
+ scanned_owner_ids.add(owner_id)
+ sources.update(owner_sources)
+ source_keys.update(owner_sources)
self._sources = sources
- await asyncio.to_thread(self.repo.reconcile_sources, source_keys)
+ await asyncio.to_thread(
+ self.repo.reconcile_sources, source_keys, scanned_owner_ids=scanned_owner_ids
+ )
def _start_due_jobs(self) -> None:
for job in self.repo.due_jobs():
key = self.repo.key(job)
- if key in self._run_tasks:
+ if key in self._run_tasks or key not in self._sources:
continue
if len(self._run_tasks) >= self._max_concurrency:
return
@@ -193,18 +198,20 @@ class WebSourceSyncScheduler:
from deeptutor.services.web_source.sync import sync_source
key = self.repo.key(scheduled)
- lease_until = _now_ms() + _hours_ms(WEB_SYNC_LEASE_HOURS)
+ lease_until = _now_ms() + WEB_SYNC_LEASE_SECONDS * 1000
claimed = await asyncio.to_thread(
self.repo.claim,
scheduled,
- runner_id=self._runner_id,
+ runner_id=f"{self._runner_id}:{uuid.uuid4().hex}",
lease_until_ms=lease_until,
)
if claimed is None:
return
source = self._sources.get(key)
if source is None:
- self.repo.mark_cancelled(claimed)
+ # A scan may have failed after this job was queued. Keep the job
+ # retryable until its owner's inventory is read successfully.
+ self.repo.mark_interrupted(claimed)
return
user = (
@@ -220,6 +227,11 @@ class WebSourceSyncScheduler:
if user.scope.kind == "user":
ensure_scope_workspace(user.scope)
token = set_current_user(user)
+ owner_task = asyncio.current_task()
+ assert owner_task is not None
+ lease_task = asyncio.create_task(
+ self._renew_lease(claimed, owner_task), name=f"web-source-sync:lease:{key[2]}"
+ )
try:
current = await asyncio.to_thread(self.repo.get, key)
if current is not None and current.cancel_requested:
@@ -242,14 +254,30 @@ class WebSourceSyncScheduler:
else:
self._mark_failure(claimed, source, result.error or "Web synchronization failed")
except asyncio.CancelledError:
- self.repo.mark_cancelled(claimed, _now_ms())
+ current = self.repo.get(key)
+ if current is not None and current.cancel_requested:
+ self.repo.mark_cancelled(claimed, _now_ms())
+ else:
+ self.repo.mark_interrupted(claimed)
raise
except Exception as exc:
logger.exception("Scheduled web source sync failed for %s", source.get("url"))
self._mark_failure(claimed, source, str(exc))
finally:
+ lease_task.cancel()
+ await asyncio.gather(lease_task, return_exceptions=True)
reset_current_user(token)
+ async def _renew_lease(self, job: WebSourceSyncJob, owner_task: asyncio.Task[None]) -> None:
+ while True:
+ await asyncio.sleep(WEB_SYNC_LEASE_RENEW_SECONDS)
+ renewed = await asyncio.to_thread(
+ self.repo.renew_lease, job, _now_ms() + WEB_SYNC_LEASE_SECONDS * 1000
+ )
+ if not renewed:
+ owner_task.cancel()
+ return
+
def _mark_failure(
self,
job: WebSourceSyncJob,
diff --git a/tests/services/web_source/test_sync_scheduler.py b/tests/services/web_source/test_sync_scheduler.py
index 71348a3b1..6576d08cd 100644
--- a/tests/services/web_source/test_sync_scheduler.py
+++ b/tests/services/web_source/test_sync_scheduler.py
@@ -121,7 +121,7 @@ async def test_scheduler_cancelled_run_is_not_marked_successful(tmp_path: Path)
persisted = repository.get((job.owner_id, job.kb_name, job.source_id))
assert persisted is not None
- assert persisted.state == "cancelled"
+ assert persisted.state == "interrupted"
@pytest.mark.asyncio
@@ -147,3 +147,84 @@ async def test_scheduler_ignores_manual_and_disabled_sources(tmp_path: Path) ->
assert [job.source_id for job in scheduler.repo.list_jobs("local-admin", "kb")] == [
automatic["id"]
]
+
+
+def test_reconcile_preserves_jobs_for_an_owner_that_failed_to_scan(tmp_path: Path) -> None:
+ repository = SQLiteWebSourceSyncRepository(tmp_path / "jobs.sqlite")
+ old = ("local-admin", "kb", "old")
+ repository.reconcile_sources({old})
+ repository.reconcile_sources(set(), scanned_owner_ids=set())
+ assert repository.get(old) is not None
+ repository.reconcile_sources(set(), scanned_owner_ids={"local-admin"})
+ assert repository.get(old) is None
+
+
+def test_expired_owner_cannot_overwrite_a_new_claim(tmp_path: Path, monkeypatch) -> None:
+ repository = SQLiteWebSourceSyncRepository(tmp_path / "jobs.sqlite")
+ now = [1_000]
+ monkeypatch.setattr(repository, "_now_ms", lambda: now[0])
+ key = ("local-admin", "kb", "source")
+ repository.reconcile_sources({key})
+ old = repository.claim(repository.get(key), runner_id="runner-old", lease_until_ms=2_000)
+ assert old is not None
+ assert repository.renew_lease(old, 3_000) is True
+ now[0] = 3_001
+ assert repository.renew_lease(old, 4_000) is False
+ repository.recover_interrupted("runner-new")
+ new = repository.claim(repository.get(key), runner_id="runner-new", lease_until_ms=4_000)
+ assert new is not None
+ repository.mark_success(old, 9_000)
+ assert repository.get(key).runner_id == "runner-new"
+ repository.mark_success(new, 9_000)
+ assert repository.get(key).state == "pending"
+
+
+async def test_failed_owner_enumeration_does_not_delete_persisted_job(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ repository = SQLiteWebSourceSyncRepository(tmp_path / "jobs.sqlite")
+ key = ("local-admin", "kb", "source")
+ repository.reconcile_sources({key})
+ scheduler = WebSourceSyncScheduler(repository=repository)
+
+ class BrokenManager:
+ def get_all_web_sources(self):
+ raise RuntimeError("temporary read failure")
+
+ monkeypatch.setattr(scheduler, "_manager_for_owner", lambda _owner: BrokenManager())
+ await scheduler._synchronize_sources()
+ assert repository.get(key) is not None
+ assert scheduler._sources == {}
+
+
+async def test_scheduler_renews_lease_during_a_long_sync(tmp_path: Path, monkeypatch) -> None:
+ from deeptutor.services.web_source import scheduler as scheduler_module
+
+ monkeypatch.setattr(scheduler_module, "WEB_SYNC_LEASE_SECONDS", 0.2)
+ monkeypatch.setattr(scheduler_module, "WEB_SYNC_LEASE_RENEW_SECONDS", 0.04)
+ manager = KnowledgeBaseManager(base_dir=str(tmp_path / "kbs"))
+ (manager.base_dir / "kb").mkdir()
+ (manager.base_dir / "kb" / "metadata.json").write_text("{}", encoding="utf-8")
+ manager.register_knowledge_base("kb")
+ manager.add_web_source("kb", "https://example.com/docs/")
+ repository = SQLiteWebSourceSyncRepository(tmp_path / "jobs.sqlite")
+ scheduler = WebSourceSyncScheduler(repository=repository, manager_factory=lambda: manager)
+ await scheduler._synchronize_sources()
+ job = repository.list_jobs("local-admin", "kb")[0]
+
+ async def slow_sync(**_kwargs):
+ await asyncio.sleep(0.35)
+ return WebSyncResult(ok=True)
+
+ with patch("deeptutor.services.web_source.sync.sync_source", slow_sync):
+ running = asyncio.create_task(scheduler._run_job(job))
+ await asyncio.sleep(0.25)
+ current = repository.get(repository.key(job))
+ assert current is not None and current.state == "running"
+ assert current.lease_until_ms is not None and current.lease_until_ms > int(
+ time.time() * 1000
+ )
+ await running
+ assert repository.get(repository.key(job)).state == "pending"