DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
Preserve failed web source scans and renew fenced sync leases after #1385
Bingxi Zhao (Frank)46c9c885f7fc
diff --git a/deeptutor/services/web_source/repository.py b/deeptutor/services/web_source/repository.pyindex c1e8be855..13828fc32 100644--- a/deeptutor/services/web_source/repository.py+++ b/deeptutor/services/web_source/repository.py@@ -110,8 +110,13 @@ class SQLiteWebSourceSyncRepository:def _now_ms(self) -> int:return int(time.time() * 1000)- def reconcile_sources(self, source_keys: set[tuple[str, str, str]]) -> None:- """Create missing rows and remove jobs whose source no longer exists."""+ def reconcile_sources(+ self,+ source_keys: set[tuple[str, str, str]],+ *,+ scanned_owner_ids: set[str] | None = None,+ ) -> None:+ """Reconcile only owners whose source inventory was read successfully."""now = self._now_ms()with self._connect() as connection:connection.execute("BEGIN IMMEDIATE")@@ -134,6 +139,8 @@ class SQLiteWebSourceSyncRepository:(owner_id, kb_name, source_id, now, now),)for owner_id, kb_name, source_id in existing - source_keys:+ if scanned_owner_ids is not None and owner_id not in scanned_owner_ids:+ continueconnection.execute("""DELETE FROM web_source_sync_jobs@@ -260,27 +267,44 @@ class SQLiteWebSourceSyncRepository:connection.execute("ROLLBACK")raise+ def renew_lease(self, job: WebSourceSyncJob, lease_until_ms: int) -> bool:+ """Extend only the still-owned, unexpired claim."""+ now = self._now_ms()+ with self._connect() as connection:+ cursor = connection.execute(+ """+ UPDATE web_source_sync_jobs+ SET lease_until_ms=?, updated_at_ms=?+ WHERE owner_id=? AND kb_name=? AND source_id=?+ AND state='running' AND runner_id=? AND lease_until_ms>?+ """,+ (lease_until_ms, now, *self.key(job), job.runner_id, now),+ )+ return int(cursor.rowcount or 0) == 1+def _update(self,job: WebSourceSyncJob,**fields: Any,- ) -> None:+ ) -> bool:assignments = list(fields)if not assignments:- return+ return Falsefields["updated_at_ms"] = self._now_ms()assignments.append("updated_at_ms")values = list(fields.values())set_sql = ", ".join(f"{name}=?" for name in assignments)with self._connect() as connection:- connection.execute(+ cursor = connection.execute(f"""UPDATE web_source_sync_jobsSET {set_sql}WHERE owner_id=? AND kb_name=? AND source_id=?+ AND state='running' AND runner_id=? AND lease_until_ms>?""",- (*values, *self.key(job)),+ (*values, *self.key(job), job.runner_id, self._now_ms()),)+ return int(cursor.rowcount or 0) == 1def mark_success(self, job: WebSourceSyncJob, next_run_at_ms: int) -> None:self._update(@@ -314,6 +338,17 @@ class SQLiteWebSourceSyncRepository:lease_until_ms=None,)+ def mark_interrupted(self, job: WebSourceSyncJob) -> None:+ self._update(+ job,+ state="interrupted",+ next_run_at_ms=self._now_ms(),+ error="Synchronization was interrupted",+ cancel_requested=False,+ runner_id="",+ lease_until_ms=None,+ )+def mark_cancelled(self, job: WebSourceSyncJob, next_run_at_ms: int | None = None) -> None:now = self._now_ms()with self._connect() as connection:@@ -324,8 +359,9 @@ class SQLiteWebSourceSyncRepository:lease_until_ms=NULL, last_run_at_ms=?, updated_at_ms=?,next_run_at_ms=COALESCE(?, next_run_at_ms)WHERE owner_id=? AND kb_name=? AND source_id=?+ AND state='running' AND runner_id=? AND lease_until_ms>?""",- (now, now, next_run_at_ms, *self.key(job)),+ (now, now, next_run_at_ms, *self.key(job), job.runner_id, now),)def request_cancel(self, job_key: tuple[str, str, str]) -> bool:@@ -395,6 +431,7 @@ class SQLiteWebSourceSyncRepository:cancel_requested=0, runner_id='', lease_until_ms=NULL,updated_at_ms=?WHERE owner_id=? AND kb_name=? AND source_id=?+ AND state IN ('error', 'interrupted', 'cancelled')""",(now, now, *job_key),)diff --git a/deeptutor/services/web_source/scheduler.py b/deeptutor/services/web_source/scheduler.pyindex 84172c585..fdcac9500 100644--- a/deeptutor/services/web_source/scheduler.py+++ b/deeptutor/services/web_source/scheduler.py@@ -20,7 +20,8 @@ WEB_SYNC_MIN_INTERVAL_HOURS = 1WEB_SYNC_MAX_INTERVAL_HOURS = 168WEB_SYNC_INTERVAL_HOURS = 24WEB_SYNC_CHECK_SECONDS = 15-WEB_SYNC_LEASE_HOURS = 6+WEB_SYNC_LEASE_SECONDS = 60+WEB_SYNC_LEASE_RENEW_SECONDS = 20WEB_SYNC_MAX_CONCURRENCY = 2@@ -108,6 +109,7 @@ class WebSourceSyncScheduler:while self._running:try:await self._synchronize_sources()+ await asyncio.to_thread(self.repo.recover_interrupted, self._runner_id)self._start_due_jobs()except Exception:logger.exception("Web source scheduler cycle failed")@@ -150,30 +152,33 @@ class WebSourceSyncScheduler:del managersource_keys: set[tuple[str, str, str]] = set()sources: dict[tuple[str, str, str], dict[str, Any]] = {}+ scanned_owner_ids: set[str] = set()for owner_id in owner_ids:+ owner_sources: dict[tuple[str, str, str], dict[str, Any]] = {}try:owner_manager = await asyncio.to_thread(self._manager_for_owner, owner_id)for kb_name, source in owner_manager.get_all_web_sources():- if not source.get("enabled", True):- continue- if not source.get("auto_sync_enabled", True):+ if not source.get("enabled", True) or not source.get("auto_sync_enabled", True):continuesource_id = str(source.get("id") or "")kb_key = str(kb_name)- if not source_id or not kb_key:- continue- key = owner_id, kb_key, source_id- source_keys.add(key)- sources[key] = dict(source)+ if source_id and kb_key:+ owner_sources[owner_id, kb_key, source_id] = dict(source)except Exception:logger.exception("Failed to enumerate web sources for owner %s", owner_id)+ continue+ scanned_owner_ids.add(owner_id)+ sources.update(owner_sources)+ source_keys.update(owner_sources)self._sources = sources- await asyncio.to_thread(self.repo.reconcile_sources, source_keys)+ await asyncio.to_thread(+ self.repo.reconcile_sources, source_keys, scanned_owner_ids=scanned_owner_ids+ )def _start_due_jobs(self) -> None:for job in self.repo.due_jobs():key = self.repo.key(job)- if key in self._run_tasks:+ if key in self._run_tasks or key not in self._sources:continueif len(self._run_tasks) >= self._max_concurrency:return@@ -193,18 +198,20 @@ class WebSourceSyncScheduler:from deeptutor.services.web_source.sync import sync_sourcekey = self.repo.key(scheduled)- lease_until = _now_ms() + _hours_ms(WEB_SYNC_LEASE_HOURS)+ lease_until = _now_ms() + WEB_SYNC_LEASE_SECONDS * 1000claimed = await asyncio.to_thread(self.repo.claim,scheduled,- runner_id=self._runner_id,+ runner_id=f"{self._runner_id}:{uuid.uuid4().hex}",lease_until_ms=lease_until,)if claimed is None:returnsource = self._sources.get(key)if source is None:- self.repo.mark_cancelled(claimed)+ # A scan may have failed after this job was queued. Keep the job+ # retryable until its owner's inventory is read successfully.+ self.repo.mark_interrupted(claimed)returnuser = (@@ -220,6 +227,11 @@ class WebSourceSyncScheduler:if user.scope.kind == "user":ensure_scope_workspace(user.scope)token = set_current_user(user)+ owner_task = asyncio.current_task()+ assert owner_task is not None+ lease_task = asyncio.create_task(+ self._renew_lease(claimed, owner_task), name=f"web-source-sync:lease:{key[2]}"+ )try:current = await asyncio.to_thread(self.repo.get, key)if current is not None and current.cancel_requested:@@ -242,14 +254,30 @@ class WebSourceSyncScheduler:else:self._mark_failure(claimed, source, result.error or "Web synchronization failed")except asyncio.CancelledError:- self.repo.mark_cancelled(claimed, _now_ms())+ current = self.repo.get(key)+ if current is not None and current.cancel_requested:+ self.repo.mark_cancelled(claimed, _now_ms())+ else:+ self.repo.mark_interrupted(claimed)raiseexcept Exception as exc:logger.exception("Scheduled web source sync failed for %s", source.get("url"))self._mark_failure(claimed, source, str(exc))finally:+ lease_task.cancel()+ await asyncio.gather(lease_task, return_exceptions=True)reset_current_user(token)+ async def _renew_lease(self, job: WebSourceSyncJob, owner_task: asyncio.Task[None]) -> None:+ while True:+ await asyncio.sleep(WEB_SYNC_LEASE_RENEW_SECONDS)+ renewed = await asyncio.to_thread(+ self.repo.renew_lease, job, _now_ms() + WEB_SYNC_LEASE_SECONDS * 1000+ )+ if not renewed:+ owner_task.cancel()+ return+def _mark_failure(self,job: WebSourceSyncJob,diff --git a/tests/services/web_source/test_sync_scheduler.py b/tests/services/web_source/test_sync_scheduler.pyindex 71348a3b1..6576d08cd 100644--- a/tests/services/web_source/test_sync_scheduler.py+++ b/tests/services/web_source/test_sync_scheduler.py@@ -121,7 +121,7 @@ async def test_scheduler_cancelled_run_is_not_marked_successful(tmp_path: Path)persisted = repository.get((job.owner_id, job.kb_name, job.source_id))assert persisted is not None- assert persisted.state == "cancelled"+ assert persisted.state == "interrupted"@pytest.mark.asyncio@@ -147,3 +147,84 @@ async def test_scheduler_ignores_manual_and_disabled_sources(tmp_path: Path) ->assert [job.source_id for job in scheduler.repo.list_jobs("local-admin", "kb")] == [automatic["id"]]+++def test_reconcile_preserves_jobs_for_an_owner_that_failed_to_scan(tmp_path: Path) -> None:+ repository = SQLiteWebSourceSyncRepository(tmp_path / "jobs.sqlite")+ old = ("local-admin", "kb", "old")+ repository.reconcile_sources({old})+ repository.reconcile_sources(set(), scanned_owner_ids=set())+ assert repository.get(old) is not None+ repository.reconcile_sources(set(), scanned_owner_ids={"local-admin"})+ assert repository.get(old) is None+++def test_expired_owner_cannot_overwrite_a_new_claim(tmp_path: Path, monkeypatch) -> None:+ repository = SQLiteWebSourceSyncRepository(tmp_path / "jobs.sqlite")+ now = [1_000]+ monkeypatch.setattr(repository, "_now_ms", lambda: now[0])+ key = ("local-admin", "kb", "source")+ repository.reconcile_sources({key})+ old = repository.claim(repository.get(key), runner_id="runner-old", lease_until_ms=2_000)+ assert old is not None+ assert repository.renew_lease(old, 3_000) is True+ now[0] = 3_001+ assert repository.renew_lease(old, 4_000) is False+ repository.recover_interrupted("runner-new")+ new = repository.claim(repository.get(key), runner_id="runner-new", lease_until_ms=4_000)+ assert new is not None+ repository.mark_success(old, 9_000)+ assert repository.get(key).runner_id == "runner-new"+ repository.mark_success(new, 9_000)+ assert repository.get(key).state == "pending"+++async def test_failed_owner_enumeration_does_not_delete_persisted_job(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ repository = SQLiteWebSourceSyncRepository(tmp_path / "jobs.sqlite")+ key = ("local-admin", "kb", "source")+ repository.reconcile_sources({key})+ scheduler = WebSourceSyncScheduler(repository=repository)++ class BrokenManager:+ def get_all_web_sources(self):+ raise RuntimeError("temporary read failure")++ monkeypatch.setattr(scheduler, "_manager_for_owner", lambda _owner: BrokenManager())+ await scheduler._synchronize_sources()+ assert repository.get(key) is not None+ assert scheduler._sources == {}+++async def test_scheduler_renews_lease_during_a_long_sync(tmp_path: Path, monkeypatch) -> None:+ from deeptutor.services.web_source import scheduler as scheduler_module++ monkeypatch.setattr(scheduler_module, "WEB_SYNC_LEASE_SECONDS", 0.2)+ monkeypatch.setattr(scheduler_module, "WEB_SYNC_LEASE_RENEW_SECONDS", 0.04)+ manager = KnowledgeBaseManager(base_dir=str(tmp_path / "kbs"))+ (manager.base_dir / "kb").mkdir()+ (manager.base_dir / "kb" / "metadata.json").write_text("{}", encoding="utf-8")+ manager.register_knowledge_base("kb")+ manager.add_web_source("kb", "https://example.com/docs/")+ repository = SQLiteWebSourceSyncRepository(tmp_path / "jobs.sqlite")+ scheduler = WebSourceSyncScheduler(repository=repository, manager_factory=lambda: manager)+ await scheduler._synchronize_sources()+ job = repository.list_jobs("local-admin", "kb")[0]++ async def slow_sync(**_kwargs):+ await asyncio.sleep(0.35)+ return WebSyncResult(ok=True)++ with patch("deeptutor.services.web_source.sync.sync_source", slow_sync):+ running = asyncio.create_task(scheduler._run_job(job))+ await asyncio.sleep(0.25)+ current = repository.get(repository.key(job))+ assert current is not None and current.state == "running"+ assert current.lease_until_ms is not None and current.lease_until_ms > int(+ time.time() * 1000+ )+ await running+ assert repository.get(repository.key(job)).state == "pending"